feat(analytics): доходы, ребалансировка, налоги, бенчмарки и цели — фаза 4
Второй источник выплат: sources/tinvest/sync_events.py (GetDividends, GetBondCoupons, GetBondEvents) и sources/moex/payouts.py (ISS bondization + dividends). Приоритет между ними — pricing/payouts.resolve_payouts, решается на чтении, а не на записи: corporate_action уникален по (instrument_id, kind, source, source_id), обе версии сосуществуют, и правило можно поменять без ресинка истории. Амортизация от MOEX идёт в bond_nominal_schedule, а не в corporate_action — этим типом безраздельно владеет ledger/corporate_actions.py. analytics/income.py — metric_income_monthly (факт) и metric_income_calendar (прошлое и прогноз) с basis paid/announced/history на каждой строке, три источника числа не смешиваются. analytics/rebalance.py — сделки по portfolio_target пропорционально внутри бакета, лоты только вниз, покупки не занимают у ещё не свершившихся продаж. analytics/tax.py — оценка, не замена справки брокера: дивиденды/купоны gross, реализованный результат из lot_disposal с переоценкой каждой ноги на свою дату. analytics/benchmarks.py — TWR индекса на сетке портфеля, kind (price/total_return) не скрывается. analytics/goals.py — прогресс цели и нужный взнос по trailing XIRR. Четыре шага зарегистрированы в register_steps: benchmarks после returns (общая сетка дат), rebalance после allocation (её веса, не пересчитывает), income и tax после lots (нужен lot_disposal).
This commit is contained in:
Dmitry
@@ -103,14 +103,18 @@ def register_steps() -> None:
|
||||
|
||||
from fintracker.analytics import (
|
||||
allocation,
|
||||
benchmarks,
|
||||
cashflow,
|
||||
cashflow_broker,
|
||||
classify,
|
||||
income,
|
||||
networth,
|
||||
quality,
|
||||
rebalance,
|
||||
returns,
|
||||
runway,
|
||||
spending,
|
||||
tax,
|
||||
valuation,
|
||||
)
|
||||
from fintracker.ledger.corporate_actions import rebuild_corporate_actions
|
||||
@@ -132,8 +136,15 @@ def register_steps() -> None:
|
||||
# valuation prices the positions the lots describe; returns reads the series it writes
|
||||
register_step("valuation", valuation.rebuild_valuation)
|
||||
register_step("returns", returns.rebuild_returns)
|
||||
# benchmarks reads metric_returns to sit on the same date grid
|
||||
register_step("benchmarks", benchmarks.rebuild_benchmark_returns)
|
||||
register_step("allocation", allocation.rebuild_allocation)
|
||||
# rebalance reads metric_allocation, never re-derives a weight itself
|
||||
register_step("rebalance", rebalance.rebuild_rebalance)
|
||||
register_step("cashflow_broker", cashflow_broker.rebuild_cash_flow_broker)
|
||||
# income and tax both read event/corporate_action/lot_disposal, all written by now
|
||||
register_step("income", income.rebuild_income)
|
||||
register_step("tax", tax.rebuild_tax_year)
|
||||
register_step("networth", networth.rebuild_net_worth_daily)
|
||||
register_step("cashflow", cashflow.rebuild_cash_flow_monthly)
|
||||
register_step("spending", spending.rebuild_spending_by_category)
|
||||
|
||||
@@ -0,0 +1,256 @@
|
||||
"""A benchmark's time-weighted return, on the portfolio's own grid of days (plan §4, фаза 4).
|
||||
|
||||
An index quoted on MOEX is an ordinary `instrument`: `sources/moex` fetches its history from
|
||||
the ISS endpoint the same way it fetches a share's, and it lands in `price_daily`. Nothing
|
||||
is seeded from code — the list of benchmarks is data in the `benchmark` table, because which
|
||||
index a portfolio is measured against is the user's decision. The codes expected there are
|
||||
MOEX secids:
|
||||
|
||||
* ``IMOEX`` — the price index of the Moscow Exchange, ``kind = "price"``;
|
||||
* ``MCFTR`` — its gross total-return twin, ``kind = "total_return"``;
|
||||
* ``RGBITR`` — the government-bond total-return index, for a bond-heavy portfolio.
|
||||
|
||||
**Why the kind matters more than it looks.** IMOEX drops by the dividend on every ex-date
|
||||
and never gets it back, so over a decade it trails MCFTR by roughly the dividend yield
|
||||
compounded — for the Russian market, several percent a year. A holder of the same basket
|
||||
received those dividends. Comparing a portfolio's TWR (which includes the income it earned)
|
||||
against a price index therefore flatters the portfolio by construction, every single year,
|
||||
with no market view behind it. MCFTR is the honest opponent: both sides count dividends.
|
||||
This module does not "fix" a price index by adding a dividend estimate — that would be an
|
||||
invented number. It reports `kind` outward so the client can mark the comparison for what it
|
||||
is.
|
||||
|
||||
**Why the grid must be shared.** A return is a product of daily factors, and a factor only
|
||||
exists for a day both sides have a value for. If the index is chained over its own trading
|
||||
days while the portfolio is chained over every calendar day, the two numbers answer
|
||||
different questions and their difference is not an excess return. So the days come from
|
||||
`metric_portfolio_value_daily` for the same scope, over the same `date_from`/`date_to` that
|
||||
`metric_returns` recorded, and a day the index has no quote for is **counted**, not hidden:
|
||||
`days_skipped` is reported on both sides and the client must show it when it is non-zero.
|
||||
|
||||
A skipped day does not lose the market move. The previous quote stays as the base, so the
|
||||
next quoted day links back to it and the chain still telescopes to `p_end / p_start`. What
|
||||
`days_skipped` says is narrower and exactly true: on that many days of the compared window
|
||||
the index had nothing to say, so the two series are not day-for-day comparable.
|
||||
|
||||
There are no external flows in an index, which is why this is much simpler than
|
||||
`analytics/returns.py`: the TWR of an index is just its price change.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import delete, insert, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS
|
||||
from fintracker.analytics.returns import annualize
|
||||
from fintracker.models import (
|
||||
Benchmark,
|
||||
MetricBenchmarkReturns,
|
||||
MetricPortfolioValueDaily,
|
||||
MetricReturns,
|
||||
PriceDaily,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ZERO = Decimal(0)
|
||||
ONE = Decimal(1)
|
||||
RATE_PLACES = Decimal("0.000001")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class IndexChain:
|
||||
"""An index's cumulative return and how much of the compared window it covers."""
|
||||
|
||||
value: Decimal | None
|
||||
days_used: int
|
||||
days_skipped: int
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# pure core
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def opening_price(prices: Mapping[date, Decimal], opening: date) -> Decimal | None:
|
||||
"""The index's level at the start of the period: its last quote on or before `opening`.
|
||||
|
||||
The first day of a period is a closing value, and periods start on a calendar date that
|
||||
is often a weekend — `1y` back from a Sunday is a Sunday. Refusing to start there would
|
||||
leave a whole period unmeasured over a detail of the calendar, so the last known close is
|
||||
used. It is not a substituted number: it is the level the index actually stood at.
|
||||
"""
|
||||
if opening in prices:
|
||||
return prices[opening]
|
||||
earlier = [d for d in prices if d < opening]
|
||||
return prices[max(earlier)] if earlier else None
|
||||
|
||||
|
||||
def index_twr(
|
||||
prices: Mapping[date, Decimal], grid: Sequence[date], *, opening: Decimal | None
|
||||
) -> IndexChain:
|
||||
"""Chain the index's daily price factors over exactly the days in `grid`.
|
||||
|
||||
`grid` is the portfolio's own set of days inside the period (the opening day excluded,
|
||||
as in `returns.twr`). A day with no quote contributes no factor and increments
|
||||
`days_skipped`; the base price is kept, so the move that eventually shows up is credited
|
||||
to the first quoted day after the gap rather than lost.
|
||||
"""
|
||||
if opening is None or opening <= ZERO:
|
||||
return IndexChain(None, 0, len(grid))
|
||||
factor = ONE
|
||||
previous = opening
|
||||
used = skipped = 0
|
||||
for d in grid:
|
||||
price = prices.get(d)
|
||||
if price is None:
|
||||
skipped += 1
|
||||
continue
|
||||
if previous > ZERO:
|
||||
factor *= price / previous
|
||||
used += 1
|
||||
previous = price
|
||||
if not used:
|
||||
return IndexChain(None, 0, skipped)
|
||||
return IndexChain((factor - ONE).quantize(RATE_PLACES), used, skipped)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# I/O
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def rebuild_benchmark_returns(session: AsyncSession) -> None:
|
||||
"""Replace `metric_benchmark_returns` for every active benchmark and every metric row."""
|
||||
await session.execute(delete(MetricBenchmarkReturns))
|
||||
|
||||
found = await session.execute(
|
||||
select(Benchmark).where(Benchmark.is_active.is_(True)).order_by(Benchmark.id)
|
||||
)
|
||||
benchmarks = list(found.scalars().all())
|
||||
if not benchmarks:
|
||||
return
|
||||
|
||||
priced = [b for b in benchmarks if b.instrument_id is not None]
|
||||
silent = [b.code for b in benchmarks if b.instrument_id is None]
|
||||
if silent:
|
||||
FINDINGS.add(
|
||||
"benchmark_no_instrument",
|
||||
"warn",
|
||||
"Бенчмарки без привязанного инструмента, истории нет: " + ", ".join(sorted(silent)),
|
||||
count=len(silent),
|
||||
)
|
||||
if not priced:
|
||||
return
|
||||
|
||||
prices = await _load_prices(session, {b.instrument_id for b in priced if b.instrument_id})
|
||||
grids = await _load_grids(session)
|
||||
periods = (
|
||||
await session.execute(
|
||||
select(
|
||||
MetricReturns.scope,
|
||||
MetricReturns.period,
|
||||
MetricReturns.date_from,
|
||||
MetricReturns.date_to,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
skipped_total = 0
|
||||
for p in periods:
|
||||
grid = [d for d in grids.get(p.scope, ()) if p.date_from < d <= p.date_to]
|
||||
if not grid:
|
||||
continue
|
||||
for benchmark in priced:
|
||||
series = prices.get(benchmark.instrument_id or 0, {})
|
||||
chain = index_twr(
|
||||
series, grid, opening=opening_price(series, p.date_from) if series else None
|
||||
)
|
||||
skipped_total += chain.days_skipped
|
||||
rows.append(
|
||||
{
|
||||
"scope": p.scope,
|
||||
"period": p.period,
|
||||
"benchmark_id": benchmark.id,
|
||||
"date_from": p.date_from,
|
||||
"date_to": p.date_to,
|
||||
"twr": chain.value,
|
||||
# annualised over the CALENDAR span, not over the quoted days: an index
|
||||
# trades ~250 days a year, so scaling by its own day count would treat a
|
||||
# full year as eight months and inflate every long-period number
|
||||
"twr_annualized": annualize(chain.value, (p.date_to - p.date_from).days),
|
||||
"days_skipped": chain.days_skipped,
|
||||
}
|
||||
)
|
||||
|
||||
if rows:
|
||||
await session.execute(insert(MetricBenchmarkReturns), rows)
|
||||
_report(rows, skipped_total)
|
||||
log.info("benchmarks: %s rows over %s benchmarks", len(rows), len(priced))
|
||||
|
||||
|
||||
async def _load_prices(
|
||||
session: AsyncSession, instrument_ids: set[int]
|
||||
) -> dict[int, dict[date, Decimal]]:
|
||||
"""Each index's closes by day, exactly as quoted — no carry-forward.
|
||||
|
||||
Carrying a close forward here would erase the very thing `days_skipped` exists to report:
|
||||
a holiday would silently become a day with a return of zero.
|
||||
"""
|
||||
if not instrument_ids:
|
||||
return {}
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(PriceDaily.instrument_id, PriceDaily.d, PriceDaily.close).where(
|
||||
PriceDaily.instrument_id.in_(instrument_ids)
|
||||
)
|
||||
)
|
||||
).all()
|
||||
out: dict[int, dict[date, Decimal]] = defaultdict(dict)
|
||||
for r in rows:
|
||||
out[r.instrument_id][r.d] = Decimal(r.close)
|
||||
return dict(out)
|
||||
|
||||
|
||||
async def _load_grids(session: AsyncSession) -> dict[str, list[date]]:
|
||||
"""The days each scope was valued on — the same spine `analytics/returns.py` chained."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(MetricPortfolioValueDaily.scope, MetricPortfolioValueDaily.d).order_by(
|
||||
MetricPortfolioValueDaily.scope, MetricPortfolioValueDaily.d
|
||||
)
|
||||
)
|
||||
).all()
|
||||
out: dict[str, list[date]] = defaultdict(list)
|
||||
for r in rows:
|
||||
out[r.scope].append(r.d)
|
||||
return dict(out)
|
||||
|
||||
|
||||
def _report(rows: Sequence[Mapping[str, object]], skipped_total: int) -> None:
|
||||
"""Say out loud when a comparison is not day-for-day."""
|
||||
empty = sum(1 for r in rows if r["twr"] is None)
|
||||
if empty:
|
||||
FINDINGS.add(
|
||||
"benchmark_no_history",
|
||||
"warn",
|
||||
f"У бенчмарков нет котировок на {empty} сравнений — сравнить не с чем",
|
||||
count=empty,
|
||||
)
|
||||
if skipped_total:
|
||||
FINDINGS.add(
|
||||
"benchmark_days_skipped",
|
||||
"info",
|
||||
f"Дней без котировки индекса в сравниваемых окнах: {skipped_total} — "
|
||||
"сетка портфеля и индекса совпадает не полностью",
|
||||
count=skipped_total,
|
||||
)
|
||||
@@ -0,0 +1,436 @@
|
||||
"""How far a goal has come, and when it gets there (plan §Фаза 4).
|
||||
|
||||
A goal is an amount, a scope and optionally a date. Everything on the screen is derived:
|
||||
where the scope stands today (`metric_portfolio_value_daily`), what it has been earning
|
||||
(`metric_returns`), and what the user said they would keep adding
|
||||
(`goal.monthly_contribution`).
|
||||
|
||||
**The projected date is allowed to be NULL, and often should be.** A portfolio that is flat
|
||||
or falling and receives nothing new never reaches a target above its current value. The
|
||||
honest output for that is «not on this trend», not a date in 2087 — a far date looks like an
|
||||
answer without being one, and a client cannot tell the two apart. So the projection runs a
|
||||
month at a time over a bounded horizon (`MAX_HORIZON_MONTHS`) and returns NULL if the target
|
||||
is still out of reach at the end of it.
|
||||
|
||||
**Which rate is assumed, and when a rate is assumed at all** — that is what `basis` names:
|
||||
|
||||
* `xirr` — the scope's own trailing money-weighted return, taken from `metric_returns`.
|
||||
Only from a period spanning at least `MIN_XIRR_HISTORY_DAYS` (180): an annualised rate is
|
||||
a per-year figure, and a goal projection compounds it over years, so a rate extracted from
|
||||
six weeks of a single position moving would be multiplied into a decade of fiction. (The
|
||||
30-day floor `analytics/returns.py` uses for a per-instrument XIRR is a display threshold;
|
||||
this one feeds a forecast, which is a stronger claim.) Among the qualifying periods the
|
||||
**shortest** is preferred, because a goal is about what the portfolio is doing now, not in
|
||||
2019 — `1y` before `3y` before `all`.
|
||||
* `contribution` — no usable trailing return, but the user plans to add money every month.
|
||||
The projection then assumes zero growth: what is planned is a deposit, not a yield.
|
||||
* `none` — neither. There is no trend to extrapolate, so `projected_date` is NULL.
|
||||
|
||||
A negative trailing XIRR is used as it is, not clamped to zero. If the goal also receives
|
||||
contributions it may still be reached, slower; if it does not, the projection returns NULL,
|
||||
which is the requirement.
|
||||
|
||||
**`monthly_needed_rub`** answers the other direction: given the deadline, the current value
|
||||
and the assumed rate, how much has to go in every month. NULL without a deadline, and NULL
|
||||
once the deadline has passed — there is no «per month» in zero months left.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import delete, insert, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS, today_local
|
||||
from fintracker.models import (
|
||||
Goal,
|
||||
MetricGoalProgress,
|
||||
MetricPortfolioValueDaily,
|
||||
MetricReturns,
|
||||
)
|
||||
from fintracker.pricing.fx import FxTable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ZERO = Decimal(0)
|
||||
ONE = Decimal(1)
|
||||
MONTHS_IN_YEAR = Decimal(12)
|
||||
RATE_PLACES = Decimal("0.000001")
|
||||
MONEY_PLACES = Decimal("0.01")
|
||||
|
||||
MIN_XIRR_HISTORY_DAYS = 180
|
||||
"""Shortest trailing window whose annualised return may be compounded into a forecast."""
|
||||
|
||||
MAX_HORIZON_MONTHS = 360
|
||||
"""30 years. Past this the projection stops being information and starts being a shrug, so
|
||||
the answer becomes NULL instead of a date nobody would plan around."""
|
||||
|
||||
#: Preference order among the periods `metric_returns` stores: the recent trend first.
|
||||
PERIOD_PREFERENCE = ("1y", "3y", "all", "ytd", "6m", "3m", "1m")
|
||||
|
||||
BASIS_XIRR = "xirr"
|
||||
BASIS_CONTRIBUTION = "contribution"
|
||||
BASIS_NONE = "none"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Projection:
|
||||
projected_date: date | None
|
||||
basis: str
|
||||
assumed_rate: Decimal | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class GoalProgress:
|
||||
goal_id: int
|
||||
as_of: date
|
||||
current_value_rub: Decimal
|
||||
target_amount_rub: Decimal
|
||||
progress: Decimal
|
||||
projected_date: date | None
|
||||
basis: str
|
||||
assumed_rate: Decimal | None
|
||||
monthly_needed_rub: Decimal | None
|
||||
on_track: bool | None
|
||||
|
||||
|
||||
def monthly_rate(annual: Decimal) -> Decimal:
|
||||
"""The monthly rate that compounds to `annual` over twelve months.
|
||||
|
||||
Decimal all the way: `(1 + r) ** (1/12)` is exactly what `decimal` computes correctly
|
||||
rounded, so no float ever touches a number that ends up in a stored column.
|
||||
"""
|
||||
base = ONE + annual
|
||||
if base <= ZERO:
|
||||
# a loss of 100 % or worse has no real twelfth root; treat it as total loss
|
||||
return Decimal(-1)
|
||||
return base ** (ONE / MONTHS_IN_YEAR) - ONE
|
||||
|
||||
|
||||
def add_months(d: date, months: int) -> date:
|
||||
total = d.year * 12 + (d.month - 1) + months
|
||||
year, month = divmod(total, 12)
|
||||
day = min(d.day, _days_in_month(year, month + 1))
|
||||
return date(year, month + 1, day)
|
||||
|
||||
|
||||
def _days_in_month(year: int, month: int) -> int:
|
||||
if month == 12:
|
||||
return 31
|
||||
return (date(year, month + 1, 1) - date(year, month, 1)).days
|
||||
|
||||
|
||||
def months_between(start: date, end: date) -> int:
|
||||
"""Whole months from `start` to `end`, never negative."""
|
||||
months = (end.year - start.year) * 12 + (end.month - start.month)
|
||||
if end.day < start.day:
|
||||
months -= 1
|
||||
return max(0, months)
|
||||
|
||||
|
||||
def project(
|
||||
*,
|
||||
current: Decimal,
|
||||
target: Decimal,
|
||||
as_of: date,
|
||||
annual_rate: Decimal | None,
|
||||
monthly_contribution: Decimal,
|
||||
basis: str,
|
||||
) -> Projection:
|
||||
"""Step the balance forward a month at a time until it reaches the target, or give up.
|
||||
|
||||
Iteration rather than a closed-form solve because the closed form needs a logarithm (a
|
||||
float) and has to special-case a zero rate, a negative rate and a contribution that
|
||||
exactly offsets the drawdown. A loop of at most 360 Decimal multiplications is cheap,
|
||||
exact, and impossible to read wrong.
|
||||
"""
|
||||
if basis == BASIS_NONE:
|
||||
return Projection(None, BASIS_NONE, None)
|
||||
if current >= target:
|
||||
return Projection(as_of, basis, annual_rate)
|
||||
|
||||
rate = annual_rate if annual_rate is not None else ZERO
|
||||
step = monthly_rate(rate)
|
||||
value = current
|
||||
for month in range(1, MAX_HORIZON_MONTHS + 1):
|
||||
value = value * (ONE + step) + monthly_contribution
|
||||
if value >= target:
|
||||
return Projection(add_months(as_of, month), basis, annual_rate)
|
||||
return Projection(None, basis, annual_rate)
|
||||
|
||||
|
||||
def monthly_needed(
|
||||
*, current: Decimal, target: Decimal, months: int, annual_rate: Decimal | None
|
||||
) -> Decimal | None:
|
||||
"""Contribution per month that lands exactly on `target` after `months` months.
|
||||
|
||||
Same iteration argument as `project`: the annuity formula divides by the rate and breaks
|
||||
at zero, so the required payment is found by bisection on a function that is monotone in
|
||||
the payment. Returns 0 when the goal is already met on the trend alone.
|
||||
"""
|
||||
if months <= 0:
|
||||
return None
|
||||
rate = annual_rate if annual_rate is not None else ZERO
|
||||
step = monthly_rate(rate)
|
||||
|
||||
def final(payment: Decimal) -> Decimal:
|
||||
value = current
|
||||
for _ in range(months):
|
||||
value = value * (ONE + step) + payment
|
||||
return value
|
||||
|
||||
if final(ZERO) >= target:
|
||||
return ZERO
|
||||
low, high = ZERO, max(target, ONE)
|
||||
while final(high) < target:
|
||||
high *= 2
|
||||
if high > target * 1000:
|
||||
return None
|
||||
for _ in range(60):
|
||||
mid = (low + high) / 2
|
||||
if final(mid) < target:
|
||||
low = mid
|
||||
else:
|
||||
high = mid
|
||||
return high.quantize(MONEY_PLACES)
|
||||
|
||||
|
||||
def pick_rate(rows: Sequence[Mapping[str, object]]) -> tuple[Decimal | None, str]:
|
||||
"""Trailing XIRR to project with, and the period it came from.
|
||||
|
||||
`rows` are `metric_returns` rows of the goal's scope as mappings with `period`,
|
||||
`date_from`, `date_to` and `xirr`.
|
||||
"""
|
||||
usable = {}
|
||||
for row in rows:
|
||||
xirr = row.get("xirr")
|
||||
date_from, date_to = row.get("date_from"), row.get("date_to")
|
||||
if xirr is None or not isinstance(date_from, date) or not isinstance(date_to, date):
|
||||
continue
|
||||
if (date_to - date_from).days < MIN_XIRR_HISTORY_DAYS:
|
||||
continue
|
||||
usable[str(row.get("period"))] = Decimal(str(xirr))
|
||||
for period in PERIOD_PREFERENCE:
|
||||
if period in usable:
|
||||
return usable[period], period
|
||||
return None, ""
|
||||
|
||||
|
||||
def evaluate(
|
||||
*,
|
||||
goal_id: int,
|
||||
as_of: date,
|
||||
current: Decimal,
|
||||
target: Decimal,
|
||||
target_date: date | None,
|
||||
monthly_contribution: Decimal | None,
|
||||
trailing_xirr: Decimal | None,
|
||||
) -> GoalProgress:
|
||||
"""Everything the progress row holds, from numbers already in hand."""
|
||||
contribution = monthly_contribution or ZERO
|
||||
if trailing_xirr is not None:
|
||||
basis, rate = BASIS_XIRR, trailing_xirr
|
||||
elif contribution > ZERO:
|
||||
basis, rate = BASIS_CONTRIBUTION, ZERO
|
||||
else:
|
||||
basis, rate = BASIS_NONE, None
|
||||
|
||||
projection = project(
|
||||
current=current,
|
||||
target=target,
|
||||
as_of=as_of,
|
||||
annual_rate=rate,
|
||||
monthly_contribution=contribution,
|
||||
basis=basis,
|
||||
)
|
||||
progress = (current / target).quantize(RATE_PLACES) if target > ZERO else ZERO
|
||||
progress = max(ZERO, progress)
|
||||
|
||||
needed: Decimal | None = None
|
||||
on_track: bool | None = None
|
||||
if target_date is not None:
|
||||
needed = monthly_needed(
|
||||
current=current,
|
||||
target=target,
|
||||
months=months_between(as_of, target_date),
|
||||
annual_rate=rate,
|
||||
)
|
||||
projected = projection.projected_date
|
||||
on_track = projected is not None and projected <= target_date
|
||||
|
||||
return GoalProgress(
|
||||
goal_id=goal_id,
|
||||
as_of=as_of,
|
||||
current_value_rub=current,
|
||||
target_amount_rub=target,
|
||||
progress=progress,
|
||||
projected_date=projection.projected_date,
|
||||
basis=projection.basis,
|
||||
assumed_rate=(
|
||||
projection.assumed_rate.quantize(RATE_PLACES)
|
||||
if projection.assumed_rate is not None
|
||||
else None
|
||||
),
|
||||
monthly_needed_rub=needed,
|
||||
on_track=on_track,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- loading
|
||||
|
||||
|
||||
async def _scope_values(session: AsyncSession) -> dict[str, Decimal]:
|
||||
"""Latest total per scope, from the daily value series."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
MetricPortfolioValueDaily.scope,
|
||||
MetricPortfolioValueDaily.d,
|
||||
MetricPortfolioValueDaily.total_rub,
|
||||
).order_by(MetricPortfolioValueDaily.scope, MetricPortfolioValueDaily.d)
|
||||
)
|
||||
).all()
|
||||
out: dict[str, Decimal] = {}
|
||||
for r in rows:
|
||||
out[r.scope] = Decimal(r.total_rub)
|
||||
return out
|
||||
|
||||
|
||||
async def _scope_rates(session: AsyncSession) -> dict[str, Decimal]:
|
||||
"""Trailing XIRR per scope, chosen by `pick_rate`."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
MetricReturns.scope,
|
||||
MetricReturns.period,
|
||||
MetricReturns.date_from,
|
||||
MetricReturns.date_to,
|
||||
MetricReturns.xirr,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
per_scope: dict[str, list[dict[str, object]]] = {}
|
||||
for r in rows:
|
||||
per_scope.setdefault(r.scope, []).append(
|
||||
{
|
||||
"period": r.period,
|
||||
"date_from": r.date_from,
|
||||
"date_to": r.date_to,
|
||||
"xirr": r.xirr,
|
||||
}
|
||||
)
|
||||
out: dict[str, Decimal] = {}
|
||||
for scope, scope_rows in per_scope.items():
|
||||
rate, _ = pick_rate(scope_rows)
|
||||
if rate is not None:
|
||||
out[scope] = rate
|
||||
return out
|
||||
|
||||
|
||||
async def compute_goal_progress(session: AsyncSession, goal: Goal) -> GoalProgress | None:
|
||||
"""Progress of one goal; None when its target cannot be expressed in RUB."""
|
||||
as_of = today_local()
|
||||
values = await _scope_values(session)
|
||||
rates = await _scope_rates(session)
|
||||
return await _evaluate_goal(session, goal, as_of, values, rates)
|
||||
|
||||
|
||||
async def _evaluate_goal(
|
||||
session: AsyncSession,
|
||||
goal: Goal,
|
||||
as_of: date,
|
||||
values: Mapping[str, Decimal],
|
||||
rates: Mapping[str, Decimal],
|
||||
) -> GoalProgress | None:
|
||||
target = Decimal(goal.target_amount)
|
||||
if (goal.currency or "RUB").upper() != "RUB":
|
||||
fx = await FxTable.load(session)
|
||||
converted = fx.to_rub(target, goal.currency, as_of)
|
||||
if converted is None:
|
||||
FINDINGS.add(
|
||||
"goal_without_fx",
|
||||
"warn",
|
||||
f"Цель «{goal.name}» задана в {goal.currency}, но курса на {as_of} нет — "
|
||||
f"прогресс не посчитан",
|
||||
ref={"goal_id": goal.id},
|
||||
)
|
||||
return None
|
||||
target = converted
|
||||
if target <= ZERO:
|
||||
FINDINGS.add(
|
||||
"goal_without_target",
|
||||
"warn",
|
||||
f"У цели «{goal.name}» неположительная сумма — прогресс не посчитан",
|
||||
ref={"goal_id": goal.id},
|
||||
)
|
||||
return None
|
||||
|
||||
current = values.get(goal.scope)
|
||||
if current is None:
|
||||
FINDINGS.add(
|
||||
"goal_scope_without_metrics",
|
||||
"warn",
|
||||
f"Для цели «{goal.name}» нет метрик по scope {goal.scope} — прогресс нулевой",
|
||||
ref={"goal_id": goal.id, "scope": goal.scope},
|
||||
)
|
||||
current = ZERO
|
||||
|
||||
return evaluate(
|
||||
goal_id=goal.id,
|
||||
as_of=as_of,
|
||||
current=current,
|
||||
target=target,
|
||||
target_date=goal.target_date,
|
||||
monthly_contribution=(
|
||||
Decimal(goal.monthly_contribution) if goal.monthly_contribution is not None else None
|
||||
),
|
||||
trailing_xirr=rates.get(goal.scope),
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- refresh step
|
||||
|
||||
|
||||
async def rebuild_goal_progress(session: AsyncSession) -> None:
|
||||
"""Replace `metric_goal_progress` for every goal that is not archived."""
|
||||
await session.execute(delete(MetricGoalProgress))
|
||||
|
||||
goals = (
|
||||
(await session.execute(select(Goal).where(Goal.archived.is_(False)).order_by(Goal.id)))
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not goals:
|
||||
return
|
||||
|
||||
as_of = today_local()
|
||||
values = await _scope_values(session)
|
||||
rates = await _scope_rates(session)
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
for goal in goals:
|
||||
progress = await _evaluate_goal(session, goal, as_of, values, rates)
|
||||
if progress is None:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"goal_id": progress.goal_id,
|
||||
"as_of": progress.as_of,
|
||||
"current_value_rub": progress.current_value_rub,
|
||||
"target_amount_rub": progress.target_amount_rub,
|
||||
"progress": progress.progress,
|
||||
"projected_date": progress.projected_date,
|
||||
"basis": progress.basis,
|
||||
"assumed_rate": progress.assumed_rate,
|
||||
"monthly_needed_rub": progress.monthly_needed_rub,
|
||||
"on_track": progress.on_track,
|
||||
}
|
||||
)
|
||||
if rows:
|
||||
await session.execute(insert(MetricGoalProgress), rows)
|
||||
log.info("goals: %s rows", len(rows))
|
||||
@@ -0,0 +1,947 @@
|
||||
"""Dividends and coupons: what already landed, and what is expected (plan §3).
|
||||
|
||||
Two tables come out of here and they answer different questions:
|
||||
|
||||
* `metric_income_monthly` — money actually received, per month, kind and currency. Pure
|
||||
history, read from `event` (confirmed only, like every metric), never from a forecast.
|
||||
* `metric_income_calendar` — one row per payment, past and future. The past rows carry
|
||||
`basis = "paid"`; the future ones carry the assumption that produced them.
|
||||
|
||||
**`basis` is the point of this module.** A bond's coupon is arithmetic on a published
|
||||
schedule; an announced dividend is a fact the issuer stated; everything else is an
|
||||
extrapolation from the last 24 months that can be wrong by any amount. They are computed by
|
||||
three different code paths on purpose, and the screen shows which one produced each number —
|
||||
a single «ожидаемый доход» that mixes them is a guess wearing a forecast's clothes.
|
||||
|
||||
What is deliberately NOT here:
|
||||
|
||||
* **No zero rows.** An instrument nobody ever paid on, and that has no schedule, does not get
|
||||
a row with `amount = 0`; it gets a `FINDINGS` warning and stays out of the forecast. A zero
|
||||
looks like knowledge — «this pays nothing» — which is the opposite of what we have.
|
||||
* **No substituted FX.** A payment whose date has no rate keeps its native amount and gets
|
||||
`amount_rub = NULL` plus a finding, exactly like the rest of the analytics. Future dates
|
||||
convert at today's rate (nobody quotes tomorrow's), which is stated rather than hidden.
|
||||
* **No position we do not hold.** The forecast multiplies by `lot.qty_remaining`, so a paper
|
||||
sold in full disappears from it and a half-sold one halves.
|
||||
|
||||
Priority between corporate-action sources (T-Invest vs MOEX) is NOT decided here. It belongs
|
||||
to `fintracker.pricing.payouts.resolve_payouts`, which every declared payout is passed
|
||||
through before it is read. The import is guarded because the two modules were written in
|
||||
parallel: when the resolver is absent we fall back to a local `paid > announced > forecast`
|
||||
preference with no cross-source rule at all. **That fallback is a stopgap, not a second
|
||||
opinion** — when `pricing/payouts.py` is importable it decides, and nothing here may grow a
|
||||
competing rule.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import calendar as _calendar
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Iterable, Mapping, Sequence
|
||||
from dataclasses import dataclass, replace
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
from itertools import pairwise
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, insert, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS, today_local
|
||||
from fintracker.analytics.valuation import account_scopes
|
||||
from fintracker.models import (
|
||||
AssetClass,
|
||||
BondNominalSchedule,
|
||||
CorporateAction,
|
||||
CorporateActionStatus,
|
||||
Event,
|
||||
EventStatus,
|
||||
IncomeBasis,
|
||||
Instrument,
|
||||
Lot,
|
||||
MetricIncomeCalendar,
|
||||
MetricIncomeMonthly,
|
||||
)
|
||||
from fintracker.models.ledger import POSITION_KINDS, EventKind
|
||||
from fintracker.pricing.fx import FxTable
|
||||
|
||||
try: # pragma: no cover - exercised by whichever half of the tree is on disk
|
||||
from fintracker.pricing.payouts import resolve_payouts # type: ignore[attr-defined]
|
||||
except ImportError: # the corporate-action sources are written in parallel; see module docstring
|
||||
resolve_payouts = None # type: ignore[assignment]
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ZERO = Decimal(0)
|
||||
RUB = "RUB"
|
||||
|
||||
#: How far ahead the calendar is built. The API caps `/income/forecast` at 36 months, so the
|
||||
#: table has to reach that far or the endpoint would answer with a silently truncated series.
|
||||
FORECAST_MONTHS = 36
|
||||
|
||||
#: How far back periodicity is inferred from. Two years is the plan's window: one is not
|
||||
#: enough to see an annual payer twice, three drags in a dividend policy that no longer holds.
|
||||
HISTORY_MONTHS = 24
|
||||
|
||||
#: Payments per year the history path is allowed to conclude (plan §3). A monthly payer is
|
||||
#: therefore rounded to quarterly — bonds avoid that by going through the schedule path, and
|
||||
#: `income_irregular_history` says so when the observed spacing does not fit.
|
||||
FREQUENCIES = (1, 2, 4)
|
||||
|
||||
#: Event kinds that are income. `repayment` is one of them: the redemption of a bond pays
|
||||
#: money, and the calendar is about money landing, not about the position surviving.
|
||||
INCOME_KINDS: tuple[EventKind, ...] = (
|
||||
EventKind.dividend,
|
||||
EventKind.coupon,
|
||||
EventKind.amortization,
|
||||
EventKind.repayment,
|
||||
)
|
||||
|
||||
#: Instruments that can pay at all — the ones a missing schedule is worth warning about.
|
||||
PAYING_CLASSES = frozenset({AssetClass.share, AssetClass.bond, AssetClass.etf, AssetClass.fund})
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# pure core: plain values in, calendar entries out — no session, no I/O
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Payment:
|
||||
"""One payout that actually happened, as the ledger recorded it."""
|
||||
|
||||
account_id: int
|
||||
instrument_id: int
|
||||
kind: str
|
||||
d: date
|
||||
currency: str
|
||||
amount: Decimal
|
||||
"""Net cash that reached the account; `tax` is already taken out of it."""
|
||||
tax: Decimal
|
||||
held_qty: Decimal
|
||||
"""Position the account held on `d` — what turns a payment into an amount per unit."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Payout:
|
||||
"""A declared or projected payout — one `corporate_action`, flattened.
|
||||
|
||||
The field set is `pricing.payouts.PayoutLike`, so a resolved row and a hand-built one are
|
||||
interchangeable and the resolver can be handed either.
|
||||
"""
|
||||
|
||||
instrument_id: int
|
||||
kind: str
|
||||
status: str
|
||||
record_date: date | None
|
||||
pay_date: date | None
|
||||
amount_per_unit: Decimal | None
|
||||
currency: str | None
|
||||
ex_date: date | None = None
|
||||
source: str = ""
|
||||
|
||||
@property
|
||||
def effective_date(self) -> date | None:
|
||||
"""When the money is expected: pay date, else ex-date, else the record date."""
|
||||
return self.pay_date or self.ex_date or self.record_date
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Entry:
|
||||
"""One calendar row before it knows which scope it belongs to."""
|
||||
|
||||
instrument_id: int
|
||||
kind: str
|
||||
expected_date: date
|
||||
record_date: date | None
|
||||
qty: Decimal
|
||||
per_unit: Decimal | None
|
||||
amount: Decimal
|
||||
currency: str
|
||||
basis: IncomeBasis
|
||||
tax_withheld: Decimal | None = None
|
||||
|
||||
|
||||
def add_months(d: date, months: int) -> date:
|
||||
"""`d` shifted by whole months, clamped to the length of the target month."""
|
||||
total = d.year * 12 + (d.month - 1) + months
|
||||
year, month = divmod(total, 12)
|
||||
month += 1
|
||||
return date(year, month, min(d.day, _calendar.monthrange(year, month)[1]))
|
||||
|
||||
|
||||
def month_start(d: date) -> date:
|
||||
return d.replace(day=1)
|
||||
|
||||
|
||||
def nominal_at(schedule: Sequence[tuple[date, Decimal]], d: date) -> Decimal | None:
|
||||
"""The nominal in force on `d`: the last entry that had already taken effect.
|
||||
|
||||
`bond_nominal_schedule` is ordered by `effective_date` and each row says what the nominal
|
||||
*becomes*, so a date before the first row has no answer here — the caller falls back to
|
||||
`instrument.nominal`.
|
||||
"""
|
||||
current: Decimal | None = None
|
||||
for effective, value in schedule:
|
||||
if effective > d:
|
||||
break
|
||||
current = value
|
||||
return current
|
||||
|
||||
|
||||
def coupon_per_unit(
|
||||
declared: Decimal, base_nominal: Decimal | None, nominal_on_date: Decimal | None
|
||||
) -> Decimal:
|
||||
"""A published coupon rescaled to the nominal that stands on the coupon's own date.
|
||||
|
||||
A coupon is a rate on the nominal, and an amortisation cuts the nominal — so a schedule
|
||||
published against the issue par overstates every coupon after the first amortisation, by
|
||||
exactly the fraction already repaid. This is the single most common way to get a bond
|
||||
forecast wrong, which is why it is one function with one test.
|
||||
|
||||
Scaling is skipped when either nominal is unknown or when they are equal (the ordinary
|
||||
non-amortised bond), so a source that already publishes amortised coupons is not cut
|
||||
twice as long as it also publishes the matching nominal schedule.
|
||||
"""
|
||||
if base_nominal is None or nominal_on_date is None or base_nominal <= ZERO:
|
||||
return declared
|
||||
if nominal_on_date == base_nominal:
|
||||
return declared
|
||||
return declared * nominal_on_date / base_nominal
|
||||
|
||||
|
||||
def detect_frequency(dates: Sequence[date]) -> int | None:
|
||||
"""Payments per year implied by `dates`, snapped to 1, 2 or 4 — or None if unknowable.
|
||||
|
||||
One payment says nothing about spacing, so it is read as annual: that is the commonest
|
||||
Russian dividend and the alternative (dropping the paper) hides a payout we have actually
|
||||
seen. Everything here is labelled `history` downstream, which is the honest part.
|
||||
|
||||
Where it breaks: a payer that changed policy inside the window (two payments a year, then
|
||||
one) averages into something it never was, and a monthly payer is rounded down to
|
||||
quarterly because the plan admits only three buckets. `regular()` is what flags the first
|
||||
case; the second only reaches here for a bond with no schedule at all.
|
||||
"""
|
||||
unique = sorted(set(dates))
|
||||
if not unique:
|
||||
return None
|
||||
if len(unique) == 1:
|
||||
return 1
|
||||
span = (unique[-1] - unique[0]).days
|
||||
if span <= 0:
|
||||
return None
|
||||
interval = Decimal(span) / Decimal(len(unique) - 1)
|
||||
per_year = Decimal(365) / interval
|
||||
return min(FREQUENCIES, key=lambda f: abs(per_year - Decimal(f)))
|
||||
|
||||
|
||||
def regular(dates: Sequence[date]) -> bool:
|
||||
"""True when the gaps between payments are within 2x of each other.
|
||||
|
||||
An irregular history still produces a forecast — refusing to guess at all would empty the
|
||||
screen for every special dividend — but it is reported, so the number carries its doubt.
|
||||
"""
|
||||
unique = sorted(set(dates))
|
||||
if len(unique) < 3:
|
||||
return True
|
||||
gaps = [(b - a).days for a, b in pairwise(unique)]
|
||||
smallest, largest = min(gaps), max(gaps)
|
||||
return smallest > 0 and Decimal(largest) <= Decimal(smallest) * 2
|
||||
|
||||
|
||||
def project_dates(last: date, frequency: int, *, start: date, end: date) -> list[date]:
|
||||
"""Future payment dates implied by a frequency, inside [start, end]."""
|
||||
step = 12 // frequency
|
||||
out: list[date] = []
|
||||
k = 1
|
||||
while True:
|
||||
d = add_months(last, step * k)
|
||||
if d > end:
|
||||
break
|
||||
if d >= start:
|
||||
out.append(d)
|
||||
k += 1
|
||||
return out
|
||||
|
||||
|
||||
def fold_payments(payments: Iterable[Payment]) -> list[Entry]:
|
||||
"""Received payouts as `paid` calendar entries, one per (instrument, kind, date, currency).
|
||||
|
||||
Folding across accounts is what makes a scope's calendar readable: the same dividend paid
|
||||
into a brokerage and an ИИС account is one event in the world, and two rows would read as
|
||||
two dividends.
|
||||
"""
|
||||
grouped: dict[tuple[int, str, date, str], list[Payment]] = defaultdict(list)
|
||||
for p in payments:
|
||||
grouped[(p.instrument_id, p.kind, p.d, p.currency)].append(p)
|
||||
|
||||
out: list[Entry] = []
|
||||
for (instrument_id, kind, d, currency), group in sorted(grouped.items()):
|
||||
amount = sum((p.amount for p in group), start=ZERO)
|
||||
tax = sum((p.tax for p in group), start=ZERO)
|
||||
qty = sum((p.held_qty for p in group), start=ZERO)
|
||||
out.append(
|
||||
Entry(
|
||||
instrument_id=instrument_id,
|
||||
kind=kind,
|
||||
expected_date=d,
|
||||
record_date=None,
|
||||
qty=qty,
|
||||
per_unit=amount / qty if qty > ZERO else None,
|
||||
amount=amount,
|
||||
currency=currency,
|
||||
basis=IncomeBasis.paid,
|
||||
tax_withheld=tax,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
MonthlyRow = tuple[Decimal, Decimal, int]
|
||||
"""(amount, tax withheld, payment count) for one (month, kind, currency)."""
|
||||
|
||||
|
||||
def monthly_rows(payments: Iterable[Payment]) -> dict[tuple[date, str, str], MonthlyRow]:
|
||||
"""(month, kind, currency) -> (amount, tax withheld, payment count)."""
|
||||
out: dict[tuple[date, str, str], MonthlyRow] = {}
|
||||
for p in payments:
|
||||
key = (month_start(p.d), p.kind, p.currency)
|
||||
amount, tax, count = out.get(key, (ZERO, ZERO, 0))
|
||||
out[key] = (amount + p.amount, tax + p.tax, count + 1)
|
||||
return out
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BondFacts:
|
||||
"""Everything the schedule path needs about one bond."""
|
||||
|
||||
nominal: Decimal | None
|
||||
"""Issue par, the base a published coupon is quoted against."""
|
||||
nominal_schedule: tuple[tuple[date, Decimal], ...]
|
||||
maturity_date: date | None
|
||||
currency: str
|
||||
|
||||
|
||||
def bond_entries(
|
||||
instrument_id: int,
|
||||
facts: BondFacts,
|
||||
coupons: Sequence[Payout],
|
||||
qty: Decimal,
|
||||
*,
|
||||
start: date,
|
||||
end: date,
|
||||
) -> list[Entry]:
|
||||
"""Coupons, amortisations and the redemption of one bond — the `schedule` basis.
|
||||
|
||||
All three come from published arithmetic rather than from history: the coupon from the
|
||||
payment schedule scaled by `coupon_per_unit`, the amortisation from the step in
|
||||
`bond_nominal_schedule` itself (the size of the step *is* the payment), the redemption
|
||||
from the maturity date and the nominal that survives to it.
|
||||
"""
|
||||
schedule = list(facts.nominal_schedule)
|
||||
base = schedule[0][1] if schedule else facts.nominal
|
||||
out: list[Entry] = []
|
||||
|
||||
for payout in coupons:
|
||||
d = payout.effective_date
|
||||
if d is None or not (start <= d <= end) or payout.amount_per_unit is None:
|
||||
continue
|
||||
per_unit = coupon_per_unit(
|
||||
Decimal(payout.amount_per_unit), base, nominal_at(schedule, d) or facts.nominal
|
||||
)
|
||||
out.append(
|
||||
Entry(
|
||||
instrument_id=instrument_id,
|
||||
kind="coupon",
|
||||
expected_date=d,
|
||||
record_date=payout.record_date,
|
||||
qty=qty,
|
||||
per_unit=per_unit,
|
||||
amount=per_unit * qty,
|
||||
currency=(payout.currency or facts.currency).upper(),
|
||||
basis=IncomeBasis.schedule,
|
||||
)
|
||||
)
|
||||
|
||||
previous: Decimal | None = None
|
||||
for effective, value in schedule:
|
||||
if previous is not None and start <= effective <= end and value < previous:
|
||||
step = previous - value
|
||||
out.append(
|
||||
Entry(
|
||||
instrument_id=instrument_id,
|
||||
kind="amortization",
|
||||
expected_date=effective,
|
||||
record_date=None,
|
||||
qty=qty,
|
||||
per_unit=step,
|
||||
amount=step * qty,
|
||||
currency=facts.currency.upper(),
|
||||
basis=IncomeBasis.schedule,
|
||||
)
|
||||
)
|
||||
previous = value
|
||||
|
||||
maturity = facts.maturity_date
|
||||
if maturity is not None and start <= maturity <= end:
|
||||
par = nominal_at(schedule, maturity) or facts.nominal
|
||||
if par is not None:
|
||||
out.append(
|
||||
Entry(
|
||||
instrument_id=instrument_id,
|
||||
kind="repayment",
|
||||
expected_date=maturity,
|
||||
record_date=None,
|
||||
qty=qty,
|
||||
per_unit=par,
|
||||
amount=par * qty,
|
||||
currency=facts.currency.upper(),
|
||||
basis=IncomeBasis.schedule,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def announced_entries(
|
||||
instrument_id: int, payouts: Sequence[Payout], qty: Decimal, *, start: date, end: date
|
||||
) -> list[Entry]:
|
||||
"""Declared payouts with a future date — a fact from the issuer, not an extrapolation."""
|
||||
out: list[Entry] = []
|
||||
for payout in payouts:
|
||||
if payout.status != CorporateActionStatus.announced.value:
|
||||
continue
|
||||
d = payout.effective_date
|
||||
if d is None or not (start <= d <= end) or payout.amount_per_unit is None:
|
||||
continue
|
||||
per_unit = Decimal(payout.amount_per_unit)
|
||||
out.append(
|
||||
Entry(
|
||||
instrument_id=instrument_id,
|
||||
kind=payout.kind,
|
||||
expected_date=d,
|
||||
record_date=payout.record_date,
|
||||
qty=qty,
|
||||
per_unit=per_unit,
|
||||
amount=per_unit * qty,
|
||||
currency=(payout.currency or RUB).upper(),
|
||||
basis=IncomeBasis.announced,
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
#: An extrapolated date this close to an announced one is the same payment, and loses.
|
||||
SAME_PAYMENT_DAYS = 45
|
||||
|
||||
|
||||
def history_entries(
|
||||
instrument_id: int,
|
||||
payments: Sequence[Payment],
|
||||
qty: Decimal,
|
||||
*,
|
||||
start: date,
|
||||
end: date,
|
||||
) -> tuple[list[Entry], bool]:
|
||||
"""Extrapolation from the last payments: the honest guess, labelled `history`.
|
||||
|
||||
Returns the entries and whether the history looked regular. The amount is the LAST
|
||||
payment's per-unit amount times the position held now — not the last total, which would
|
||||
keep paying on shares already sold.
|
||||
"""
|
||||
per_kind: dict[str, list[Payment]] = defaultdict(list)
|
||||
for p in payments:
|
||||
per_kind[p.kind].append(p)
|
||||
|
||||
out: list[Entry] = []
|
||||
steady = True
|
||||
for kind, group in sorted(per_kind.items()):
|
||||
by_date: dict[date, tuple[Decimal, Decimal, str]] = {}
|
||||
for p in group:
|
||||
amount, held, _ = by_date.get(p.d, (ZERO, ZERO, p.currency))
|
||||
by_date[p.d] = (amount + p.amount, held + p.held_qty, p.currency)
|
||||
dates = sorted(by_date)
|
||||
frequency = detect_frequency(dates)
|
||||
if frequency is None:
|
||||
continue
|
||||
steady = steady and regular(dates)
|
||||
last = dates[-1]
|
||||
amount, held, currency = by_date[last]
|
||||
per_unit = amount / held if held > ZERO else None
|
||||
if per_unit is None:
|
||||
continue
|
||||
for d in project_dates(last, frequency, start=start, end=end):
|
||||
out.append(
|
||||
Entry(
|
||||
instrument_id=instrument_id,
|
||||
kind=kind,
|
||||
expected_date=d,
|
||||
record_date=None,
|
||||
qty=qty,
|
||||
per_unit=per_unit,
|
||||
amount=per_unit * qty,
|
||||
currency=currency.upper(),
|
||||
basis=IncomeBasis.history,
|
||||
)
|
||||
)
|
||||
return out, steady
|
||||
|
||||
|
||||
def drop_shadowed(announced: Sequence[Entry], projected: Sequence[Entry]) -> list[Entry]:
|
||||
"""Extrapolated entries that an announced payout already covers are removed.
|
||||
|
||||
An issuer that has declared its autumn dividend has answered the question history was
|
||||
guessing at, so the guess goes — but the *other* payment of a twice-a-year payer, which
|
||||
nobody has declared yet, stays. Dropping every projection the moment one date is
|
||||
announced would quietly halve the year.
|
||||
"""
|
||||
if not announced:
|
||||
return list(projected)
|
||||
out: list[Entry] = []
|
||||
for entry in projected:
|
||||
clash = any(
|
||||
a.kind == entry.kind
|
||||
and abs((a.expected_date - entry.expected_date).days) <= SAME_PAYMENT_DAYS
|
||||
for a in announced
|
||||
)
|
||||
if not clash:
|
||||
out.append(entry)
|
||||
return out
|
||||
|
||||
|
||||
def _payout_day(payout: object) -> date | None:
|
||||
for attr in ("pay_date", "ex_date", "record_date"):
|
||||
value = getattr(payout, attr, None)
|
||||
if value is not None:
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def resolve_actions_fallback(payouts: Sequence[Any]) -> list[Any]:
|
||||
"""Stopgap for `pricing.payouts.resolve_payouts` (see the module docstring).
|
||||
|
||||
One payment can be described by both T-Invest and MOEX; without a rule the calendar shows
|
||||
it twice. With no resolver on disk we keep the best-known status per (instrument, kind,
|
||||
date) — `paid` over `announced` over `forecast` — and drop `cancelled` outright.
|
||||
Cross-source priority is explicitly NOT decided here, and this path exists only so that
|
||||
the income step still runs when `pricing/payouts.py` is missing.
|
||||
"""
|
||||
rank = {
|
||||
CorporateActionStatus.paid: 3,
|
||||
CorporateActionStatus.announced: 2,
|
||||
CorporateActionStatus.forecast: 1,
|
||||
}
|
||||
best: dict[tuple[int, str, date | None], Any] = {}
|
||||
for payout in payouts:
|
||||
if payout.status == CorporateActionStatus.cancelled:
|
||||
continue
|
||||
key = (payout.instrument_id, str(payout.kind), _payout_day(payout))
|
||||
current = best.get(key)
|
||||
if current is None or rank.get(payout.status, 0) > rank.get(current.status, 0):
|
||||
best[key] = payout
|
||||
return sorted(
|
||||
best.values(),
|
||||
key=lambda p: (p.instrument_id, str(p.kind), _payout_day(p) or date.min),
|
||||
)
|
||||
|
||||
|
||||
def resolve_actions(payouts: Sequence[Any]) -> list[Any]:
|
||||
"""Delegate to `pricing.payouts.resolve_payouts` when it exists, else fall back."""
|
||||
if resolve_payouts is None:
|
||||
return resolve_actions_fallback(payouts)
|
||||
try:
|
||||
return list(resolve_payouts(payouts))
|
||||
except (AttributeError, TypeError, ValueError): # not the shape we guessed at
|
||||
log.warning("resolve_payouts rejected our call shape; using the local fallback")
|
||||
return resolve_actions_fallback(payouts)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# the refresh step
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def rebuild_income(session: AsyncSession) -> None:
|
||||
"""Replace `metric_income_calendar` and `metric_income_monthly` for every scope."""
|
||||
await session.execute(delete(MetricIncomeCalendar))
|
||||
await session.execute(delete(MetricIncomeMonthly))
|
||||
|
||||
payments = await _load_payments(session)
|
||||
open_qty = await _open_quantities(session)
|
||||
accounts = {p.account_id for p in payments} | {a for a, _ in open_qty}
|
||||
accounts |= await _ledger_accounts(session)
|
||||
scopes = await account_scopes(session, accounts)
|
||||
if not scopes:
|
||||
return
|
||||
|
||||
instruments = await _load_instruments(session)
|
||||
schedules = await _load_nominal_schedules(session)
|
||||
payouts = await _load_payouts(session)
|
||||
fx = await FxTable.load(session)
|
||||
|
||||
today = today_local()
|
||||
horizon = add_months(today, FORECAST_MONTHS)
|
||||
since = add_months(today, -HISTORY_MONTHS)
|
||||
|
||||
calendar_rows: list[dict[str, object]] = []
|
||||
monthly_out: list[dict[str, object]] = []
|
||||
missing_fx: set[int] = set()
|
||||
|
||||
for scope, account_ids in sorted(scopes.items()):
|
||||
scoped = [p for p in payments if p.account_id in account_ids]
|
||||
qty_by_instrument: dict[int, Decimal] = defaultdict(lambda: ZERO)
|
||||
for (account_id, instrument_id), qty in open_qty.items():
|
||||
if account_id in account_ids:
|
||||
qty_by_instrument[instrument_id] += qty
|
||||
|
||||
entries = fold_payments(scoped)
|
||||
entries += _forecast(
|
||||
scope=scope,
|
||||
payments=[p for p in scoped if p.d >= since],
|
||||
qty_by_instrument=qty_by_instrument,
|
||||
instruments=instruments,
|
||||
schedules=schedules,
|
||||
payouts=payouts,
|
||||
today=today,
|
||||
horizon=horizon,
|
||||
)
|
||||
for entry in _dedupe(entries):
|
||||
rub = fx.to_rub(entry.amount, entry.currency, min(entry.expected_date, today))
|
||||
if rub is None:
|
||||
missing_fx.add(entry.instrument_id)
|
||||
calendar_rows.append(
|
||||
{
|
||||
"scope": scope,
|
||||
"instrument_id": entry.instrument_id,
|
||||
"kind": entry.kind,
|
||||
"expected_date": entry.expected_date,
|
||||
"record_date": entry.record_date,
|
||||
"qty": entry.qty,
|
||||
"per_unit": entry.per_unit,
|
||||
"amount": entry.amount,
|
||||
"currency": entry.currency,
|
||||
"amount_rub": rub,
|
||||
"basis": entry.basis,
|
||||
"tax_withheld": entry.tax_withheld,
|
||||
}
|
||||
)
|
||||
|
||||
for (month, kind, currency), (amount, tax, count) in sorted(monthly_rows(scoped).items()):
|
||||
rub = _month_rub(fx, scoped, month, kind, currency)
|
||||
monthly_out.append(
|
||||
{
|
||||
"scope": scope,
|
||||
"month": month,
|
||||
"kind": kind,
|
||||
"currency": currency,
|
||||
"amount": amount,
|
||||
"amount_rub": rub,
|
||||
"tax_withheld": tax,
|
||||
"payment_count": count,
|
||||
}
|
||||
)
|
||||
|
||||
if calendar_rows:
|
||||
await session.execute(insert(MetricIncomeCalendar), calendar_rows)
|
||||
if monthly_out:
|
||||
await session.execute(insert(MetricIncomeMonthly), monthly_out)
|
||||
if missing_fx:
|
||||
FINDINGS.add(
|
||||
"income_missing_fx",
|
||||
"warn",
|
||||
f"У {len(missing_fx)} инструментов выплата в валюте без курса на дату — "
|
||||
f"строка календаря есть, рублёвая сумма пустая",
|
||||
count=len(missing_fx),
|
||||
ref={"instruments": sorted(missing_fx)},
|
||||
)
|
||||
log.info("income: %s calendar rows, %s monthly rows", len(calendar_rows), len(monthly_out))
|
||||
|
||||
|
||||
def _month_rub(
|
||||
fx: FxTable, payments: Sequence[Payment], month: date, kind: str, currency: str
|
||||
) -> Decimal | None:
|
||||
"""A month's total in RUB — NULL as soon as one payment in it had no rate."""
|
||||
total = ZERO
|
||||
for p in payments:
|
||||
if month_start(p.d) != month or p.kind != kind or p.currency != currency:
|
||||
continue
|
||||
rub = fx.to_rub(p.amount, p.currency, p.d)
|
||||
if rub is None:
|
||||
return None
|
||||
total += rub
|
||||
return total
|
||||
|
||||
|
||||
def _forecast(
|
||||
*,
|
||||
scope: str,
|
||||
payments: Sequence[Payment],
|
||||
qty_by_instrument: Mapping[int, Decimal],
|
||||
instruments: Mapping[int, Instrument],
|
||||
schedules: Mapping[int, tuple[tuple[date, Decimal], ...]],
|
||||
payouts: Mapping[int, list[Payout]],
|
||||
today: date,
|
||||
horizon: date,
|
||||
) -> list[Entry]:
|
||||
"""The forward half of the calendar for one scope."""
|
||||
by_instrument: dict[int, list[Payment]] = defaultdict(list)
|
||||
for p in payments:
|
||||
by_instrument[p.instrument_id].append(p)
|
||||
|
||||
out: list[Entry] = []
|
||||
silent: list[int] = []
|
||||
irregular: list[int] = []
|
||||
|
||||
for instrument_id, qty in sorted(qty_by_instrument.items()):
|
||||
instrument = instruments.get(instrument_id)
|
||||
if instrument is None or qty <= ZERO:
|
||||
continue
|
||||
|
||||
instrument_payouts = payouts.get(instrument_id, [])
|
||||
entries: list[Entry] = []
|
||||
if instrument.asset_class == AssetClass.bond:
|
||||
facts = BondFacts(
|
||||
nominal=Decimal(instrument.nominal) if instrument.nominal is not None else None,
|
||||
nominal_schedule=schedules.get(instrument_id, ()),
|
||||
maturity_date=instrument.maturity_date,
|
||||
currency=(instrument.nominal_currency or instrument.currency or RUB),
|
||||
)
|
||||
coupons = [
|
||||
p
|
||||
for p in instrument_payouts
|
||||
if p.kind == "coupon" and p.status != CorporateActionStatus.cancelled
|
||||
]
|
||||
entries = bond_entries(instrument_id, facts, coupons, qty, start=today, end=horizon)
|
||||
|
||||
if not entries:
|
||||
announced = announced_entries(
|
||||
instrument_id, instrument_payouts, qty, start=today, end=horizon
|
||||
)
|
||||
projected, steady = history_entries(
|
||||
instrument_id,
|
||||
by_instrument.get(instrument_id, []),
|
||||
qty,
|
||||
start=today,
|
||||
end=horizon,
|
||||
)
|
||||
if not steady:
|
||||
irregular.append(instrument_id)
|
||||
entries = announced + drop_shadowed(announced, projected)
|
||||
|
||||
if not entries:
|
||||
if instrument.asset_class in PAYING_CLASSES:
|
||||
silent.append(instrument_id)
|
||||
continue
|
||||
out.extend(entries)
|
||||
|
||||
# findings are about the portfolio, not about each slice of it — only `all` reports
|
||||
if scope == "all":
|
||||
if silent:
|
||||
FINDINGS.add(
|
||||
"income_without_history",
|
||||
"warn",
|
||||
f"У {len(silent)} инструментов нет ни графика выплат, ни истории — "
|
||||
f"в прогноз доходов не вошли",
|
||||
count=len(silent),
|
||||
ref={"instruments": sorted(silent)},
|
||||
)
|
||||
if irregular:
|
||||
FINDINGS.add(
|
||||
"income_irregular_history",
|
||||
"info",
|
||||
f"У {len(irregular)} инструментов выплаты нерегулярны — "
|
||||
f"периодичность в прогнозе определена приблизительно",
|
||||
count=len(irregular),
|
||||
ref={"instruments": sorted(irregular)},
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _dedupe(entries: Sequence[Entry]) -> list[Entry]:
|
||||
"""Collapse entries onto the table's key: (instrument, kind, date, basis).
|
||||
|
||||
Two sources describing one payment, or two currencies on one day, would otherwise violate
|
||||
the unique constraint. Same-currency duplicates are summed (two accounts, one dividend);
|
||||
a genuine currency clash keeps the larger and is rare enough to leave unreported.
|
||||
"""
|
||||
merged: dict[tuple[int, str, date, IncomeBasis], Entry] = {}
|
||||
for entry in entries:
|
||||
key = (entry.instrument_id, entry.kind, entry.expected_date, entry.basis)
|
||||
current = merged.get(key)
|
||||
if current is None:
|
||||
merged[key] = entry
|
||||
continue
|
||||
if current.currency != entry.currency:
|
||||
if abs(entry.amount) > abs(current.amount):
|
||||
merged[key] = entry
|
||||
continue
|
||||
tax = None
|
||||
if current.tax_withheld is not None or entry.tax_withheld is not None:
|
||||
tax = (current.tax_withheld or ZERO) + (entry.tax_withheld or ZERO)
|
||||
merged[key] = replace(
|
||||
current,
|
||||
qty=current.qty + entry.qty,
|
||||
amount=current.amount + entry.amount,
|
||||
tax_withheld=tax,
|
||||
)
|
||||
return sorted(merged.values(), key=lambda e: (e.expected_date, e.instrument_id, e.kind))
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# loaders
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _ledger_accounts(session: AsyncSession) -> set[int]:
|
||||
return set(
|
||||
(
|
||||
await session.execute(
|
||||
select(Event.account_id).where(Event.status == EventStatus.confirmed).distinct()
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
|
||||
#: How far before a payment to look for the position, when the pay date already shows none.
|
||||
#: A dividend is earned on the record date and can arrive weeks after the shares were sold;
|
||||
#: reading only the pay date would make that payment look like it came from nowhere.
|
||||
EX_DATE_LOOKBACK_DAYS = 45
|
||||
|
||||
|
||||
PositionSeries = dict[tuple[int, int], list[tuple[date, Decimal]]]
|
||||
|
||||
|
||||
async def _position_series(session: AsyncSession) -> PositionSeries:
|
||||
"""(account, instrument) -> ordered (date, signed quantity change) from the ledger."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Event.account_id, Event.instrument_id, Event.trade_date, Event.quantity)
|
||||
.where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Event.kind.in_(tuple(POSITION_KINDS)),
|
||||
Event.instrument_id.is_not(None),
|
||||
Event.quantity.is_not(None),
|
||||
)
|
||||
.order_by(Event.trade_date)
|
||||
)
|
||||
).all()
|
||||
out: dict[tuple[int, int], list[tuple[date, Decimal]]] = defaultdict(list)
|
||||
for r in rows:
|
||||
out[(r.account_id, r.instrument_id)].append((r.trade_date, Decimal(r.quantity)))
|
||||
return out
|
||||
|
||||
|
||||
def held_at(series: Sequence[tuple[date, Decimal]], d: date) -> Decimal:
|
||||
"""Position on `d`: every quantity change up to and including that day, added up."""
|
||||
total = ZERO
|
||||
for day, delta in series:
|
||||
if day > d:
|
||||
break
|
||||
total += delta
|
||||
return total
|
||||
|
||||
|
||||
async def _load_payments(session: AsyncSession) -> list[Payment]:
|
||||
"""Every confirmed payout event — the only source the history half ever reads."""
|
||||
positions = await _position_series(session)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Event.account_id,
|
||||
Event.instrument_id,
|
||||
Event.kind,
|
||||
Event.trade_date,
|
||||
Event.currency,
|
||||
Event.amount,
|
||||
Event.tax,
|
||||
Event.quantity,
|
||||
).where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Event.kind.in_(INCOME_KINDS),
|
||||
Event.instrument_id.is_not(None),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
out: list[Payment] = []
|
||||
for r in rows:
|
||||
series = positions.get((r.account_id, r.instrument_id), [])
|
||||
held = held_at(series, r.trade_date)
|
||||
if held <= ZERO:
|
||||
held = held_at(series, r.trade_date - timedelta(days=EX_DATE_LOOKBACK_DAYS))
|
||||
out.append(
|
||||
Payment(
|
||||
account_id=r.account_id,
|
||||
instrument_id=r.instrument_id,
|
||||
kind=str(r.kind.value if hasattr(r.kind, "value") else r.kind),
|
||||
d=r.trade_date,
|
||||
currency=(r.currency or RUB).upper(),
|
||||
amount=Decimal(r.amount or 0),
|
||||
tax=Decimal(r.tax or 0),
|
||||
held_qty=max(held, ZERO),
|
||||
)
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
async def _open_quantities(session: AsyncSession) -> dict[tuple[int, int], Decimal]:
|
||||
"""Position per (account, instrument) from the open lots — the forecast's multiplier.
|
||||
|
||||
Lots, not the event replay: they are split- and amortisation-aware, and a paper that is
|
||||
fully sold simply has no open lot, which is exactly the «not in the forecast» rule.
|
||||
"""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Lot.account_id, Lot.instrument_id, Lot.qty_remaining).where(
|
||||
Lot.qty_remaining != 0
|
||||
)
|
||||
)
|
||||
).all()
|
||||
out: dict[tuple[int, int], Decimal] = defaultdict(lambda: ZERO)
|
||||
for r in rows:
|
||||
out[(r.account_id, r.instrument_id)] += Decimal(r.qty_remaining)
|
||||
return {k: v for k, v in out.items() if v != ZERO}
|
||||
|
||||
|
||||
async def _load_instruments(session: AsyncSession) -> dict[int, Instrument]:
|
||||
rows = (await session.execute(select(Instrument))).scalars().all()
|
||||
return {r.id: r for r in rows}
|
||||
|
||||
|
||||
async def _load_nominal_schedules(
|
||||
session: AsyncSession,
|
||||
) -> dict[int, tuple[tuple[date, Decimal], ...]]:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
BondNominalSchedule.instrument_id,
|
||||
BondNominalSchedule.effective_date,
|
||||
BondNominalSchedule.nominal,
|
||||
).order_by(BondNominalSchedule.instrument_id, BondNominalSchedule.effective_date)
|
||||
)
|
||||
).all()
|
||||
out: dict[int, list[tuple[date, Decimal]]] = defaultdict(list)
|
||||
for r in rows:
|
||||
out[r.instrument_id].append((r.effective_date, Decimal(r.nominal)))
|
||||
return {k: tuple(v) for k, v in out.items()}
|
||||
|
||||
|
||||
async def _load_payouts(session: AsyncSession) -> dict[int, list[Payout]]:
|
||||
"""Declared payouts per instrument, after source resolution (see the module docstring)."""
|
||||
rows = (await session.execute(select(CorporateAction))).scalars().all()
|
||||
out: dict[int, list[Payout]] = defaultdict(list)
|
||||
for r in resolve_actions(list(rows)):
|
||||
payout = Payout(
|
||||
instrument_id=r.instrument_id,
|
||||
kind=str(r.kind.value if hasattr(r.kind, "value") else r.kind),
|
||||
status=str(r.status.value if hasattr(r.status, "value") else r.status),
|
||||
record_date=r.record_date,
|
||||
pay_date=r.pay_date,
|
||||
amount_per_unit=Decimal(r.amount_per_unit) if r.amount_per_unit is not None else None,
|
||||
currency=r.currency,
|
||||
ex_date=getattr(r, "ex_date", None),
|
||||
source=getattr(r, "source", "") or "",
|
||||
)
|
||||
out[payout.instrument_id].append(payout)
|
||||
return out
|
||||
@@ -0,0 +1,603 @@
|
||||
"""Target weights versus reality, and the trades that close the gap (plan §3, фаза 4).
|
||||
|
||||
The input is a `portfolio_target` set — the user's decision — and `metric_allocation` — the
|
||||
measurement `analytics/allocation.py` has just written. Nothing here re-derives a weight:
|
||||
the four allocation dimensions already cover the same total (securities plus cash), so a
|
||||
bucket's share of the portfolio means the same thing on this screen as on the pie chart.
|
||||
|
||||
**Which paper is traded inside a bucket.** A target is set on a bucket, never on a paper.
|
||||
«Акции — 60 %» says nothing about whether the 60 % should be Сбер or Газпром, so the
|
||||
rebalancer trims and tops up **proportionally to what the bucket already holds**: every
|
||||
position keeps its share of the bucket, and only the bucket's size moves. The alternatives
|
||||
were considered and rejected:
|
||||
|
||||
* *sell the most overweight, buy the most underweight* — overweight relative to what? There
|
||||
is no per-instrument target, so the rule silently invents one (equal weight inside the
|
||||
bucket) and quietly reshapes the portfolio while claiming to only resize it;
|
||||
* *sell the biggest position first* — fewer orders, but it concentrates the whole tax event
|
||||
on one lot and drifts the bucket's composition with every rebalance.
|
||||
|
||||
Proportional is the only rule that uses nothing the user did not say. Its cost is more
|
||||
orders; whole-lot rounding cuts most of them away by itself, and a bucket whose share is too
|
||||
small to buy one lot simply gets no trade.
|
||||
|
||||
**Rounding always goes toward doing less.** Quantities are floored to whole lots, so acting
|
||||
on every suggestion can neither overshoot the target nor spend money that is not there. An
|
||||
instrument with `lot = 10` is never suggested in sevens: a recommendation that cannot be
|
||||
sent to the broker is not a recommendation.
|
||||
|
||||
**Sale proceeds do not fund purchases.** The buy side is capped by the cash that exists
|
||||
right now (`cash_available_rub`, overridable for what-if), not by cash plus whatever the
|
||||
sells would bring in. The trades are a proposal, not a sequenced plan, and money from a sale
|
||||
that has not happened is exactly the unexecutable recommendation this module exists to
|
||||
avoid. Where the cash runs out the quantity is cut to what it covers and `blocked_by_cash`
|
||||
is set, rather than the trade being dropped — the user should see what the cash is blocking.
|
||||
|
||||
**Shorts are never proposed.** A sell is capped by the position actually held (itself
|
||||
floored to whole lots), so the worst case is a position closed to zero.
|
||||
|
||||
**Tax is not modelled here.** The suggested sells ignore what they would cost in tax; by
|
||||
default we optimise the portfolio's shape, not its tax bill. That is a real omission and it
|
||||
has a direction: a lot younger than three years loses the ЛДВ exemption of art. 219.1 НК, so
|
||||
selling it is strictly more expensive than selling an older lot of the same paper, and a
|
||||
proportional trim will happily pick the young one. `analytics/tax.py` owns the per-lot
|
||||
picture (`ldv_date`, `tax_if_sold_now_rub`); until a rebalance consults it, treat a sell
|
||||
suggestion as «this much of this bucket», not «this exact lot».
|
||||
|
||||
**A bucket inside its band produces nothing.** `portfolio_target.band` is the tolerance
|
||||
around the target; within it `within_band` is true and `suggested_qty` is NULL. Without a
|
||||
band every tick of every price would produce a trade recommendation.
|
||||
|
||||
An instrument with no usable price is left out of the trades entirely and reported through
|
||||
`FINDINGS` — never a row with a zero in it, which would read as «sell nothing» rather than
|
||||
«we do not know».
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from collections.abc import Mapping, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from decimal import ROUND_FLOOR, Decimal
|
||||
|
||||
from sqlalchemy import delete, insert, select, update
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS, today_local
|
||||
from fintracker.analytics.allocation import CASH, Holding, bucket_of
|
||||
from fintracker.analytics.valuation import cash_balances
|
||||
from fintracker.models import (
|
||||
AllocationDimension,
|
||||
Instrument,
|
||||
MetricAllocation,
|
||||
MetricHolding,
|
||||
MetricRebalance,
|
||||
PortfolioAccount,
|
||||
PortfolioTarget,
|
||||
)
|
||||
from fintracker.pricing.fx import FxTable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ZERO = Decimal(0)
|
||||
ONE = Decimal(1)
|
||||
|
||||
WEIGHT_PLACES = Decimal("0.000001")
|
||||
"""Weights and drifts are fractions; six places is finer than any band anyone sets."""
|
||||
|
||||
DEFAULT_DIMENSION = AllocationDimension.asset_class
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Position:
|
||||
"""A priced holding the rebalancer may trade."""
|
||||
|
||||
instrument_id: int
|
||||
ticker: str
|
||||
name: str
|
||||
lot: int
|
||||
qty: Decimal
|
||||
"""Units held, always positive here — a short is never a rebalancing candidate."""
|
||||
unit_value_rub: Decimal
|
||||
"""Clean price of one unit in RUB. Taken from the holding's own valuation rather than
|
||||
re-converted, so this screen and the portfolio screen cannot disagree; the accrued
|
||||
interest of a bond is excluded, because it is not part of what a lot costs to buy."""
|
||||
price: Decimal
|
||||
price_currency: str
|
||||
|
||||
@property
|
||||
def value_rub(self) -> Decimal:
|
||||
return self.qty * self.unit_value_rub
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
weight: Decimal
|
||||
band: Decimal = ZERO
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Trade:
|
||||
"""One proposed order. `qty` is whole lots and within both the position and the cash."""
|
||||
|
||||
instrument_id: int
|
||||
ticker: str
|
||||
name: str
|
||||
action: str
|
||||
"""buy | sell"""
|
||||
qty: Decimal
|
||||
lot: int
|
||||
price: Decimal
|
||||
price_currency: str
|
||||
amount_rub: Decimal
|
||||
"""Magnitude of the order in RUB, always positive."""
|
||||
blocked_by_cash: bool = False
|
||||
current_value_rub: Decimal = ZERO
|
||||
current_weight: Decimal = ZERO
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BucketPlan:
|
||||
bucket: str
|
||||
current_value_rub: Decimal
|
||||
current_weight: Decimal
|
||||
target_weight: Decimal | None
|
||||
drift: Decimal | None
|
||||
within_band: bool
|
||||
delta_value_rub: Decimal
|
||||
"""Positive: this bucket should grow by this much. Negative: shrink."""
|
||||
trades: list[Trade] = field(default_factory=list)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class RebalancePlan:
|
||||
portfolio_id: int
|
||||
dimension: AllocationDimension
|
||||
as_of: date
|
||||
total_value_rub: Decimal
|
||||
cash_available_rub: Decimal
|
||||
buckets: list[BucketPlan]
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
|
||||
|
||||
def _q(value: Decimal) -> Decimal:
|
||||
return value.quantize(WEIGHT_PLACES)
|
||||
|
||||
|
||||
def _floor_lots(qty: Decimal, lot: int) -> Decimal:
|
||||
"""Largest whole number of lots that fits in `qty`, as a quantity."""
|
||||
if lot <= 0 or qty <= ZERO:
|
||||
return ZERO
|
||||
lots = (qty / Decimal(lot)).to_integral_value(rounding=ROUND_FLOOR)
|
||||
return max(ZERO, lots) * Decimal(lot)
|
||||
|
||||
|
||||
def build_plan(
|
||||
*,
|
||||
portfolio_id: int,
|
||||
dimension: AllocationDimension,
|
||||
as_of: date,
|
||||
total_value_rub: Decimal,
|
||||
bucket_values: Mapping[str, Decimal],
|
||||
positions: Mapping[str, Sequence[Position]],
|
||||
targets: Mapping[str, Target],
|
||||
cash_available_rub: Decimal,
|
||||
warnings: Sequence[str] = (),
|
||||
) -> RebalancePlan:
|
||||
"""The whole decision, with no database in sight — see the module docstring for the rules."""
|
||||
names = sorted(set(bucket_values) | set(targets))
|
||||
plans: list[BucketPlan] = []
|
||||
notes = list(warnings)
|
||||
|
||||
drafts: list[BucketPlan] = []
|
||||
for name in names:
|
||||
current = bucket_values.get(name, ZERO)
|
||||
weight = _q(current / total_value_rub) if total_value_rub > ZERO else ZERO
|
||||
target = targets.get(name)
|
||||
if target is None:
|
||||
drafts.append(
|
||||
BucketPlan(
|
||||
bucket=name,
|
||||
current_value_rub=current,
|
||||
current_weight=weight,
|
||||
target_weight=None,
|
||||
drift=None,
|
||||
within_band=False,
|
||||
delta_value_rub=ZERO,
|
||||
)
|
||||
)
|
||||
continue
|
||||
drift = _q(weight - target.weight)
|
||||
within = abs(drift) <= target.band
|
||||
delta = ZERO if within else (target.weight - weight) * total_value_rub
|
||||
drafts.append(
|
||||
BucketPlan(
|
||||
bucket=name,
|
||||
current_value_rub=current,
|
||||
current_weight=weight,
|
||||
target_weight=target.weight,
|
||||
drift=drift,
|
||||
within_band=within,
|
||||
delta_value_rub=delta,
|
||||
)
|
||||
)
|
||||
|
||||
# sells first, so the screen reads in the order the money moves; they fund nothing here
|
||||
for draft in drafts:
|
||||
if draft.delta_value_rub < ZERO:
|
||||
draft.trades.extend(_sell_side(draft, positions.get(draft.bucket, ()), notes))
|
||||
|
||||
cash_left = cash_available_rub
|
||||
for draft in sorted(drafts, key=lambda b: -b.delta_value_rub):
|
||||
if draft.delta_value_rub <= ZERO:
|
||||
continue
|
||||
trades, cash_left = _buy_side(draft, positions.get(draft.bucket, ()), cash_left, notes)
|
||||
draft.trades.extend(trades)
|
||||
|
||||
plans = sorted(drafts, key=lambda b: (-b.current_value_rub, b.bucket))
|
||||
return RebalancePlan(
|
||||
portfolio_id=portfolio_id,
|
||||
dimension=dimension,
|
||||
as_of=as_of,
|
||||
total_value_rub=total_value_rub,
|
||||
cash_available_rub=cash_available_rub,
|
||||
buckets=plans,
|
||||
warnings=notes,
|
||||
)
|
||||
|
||||
|
||||
def _shares(bucket: BucketPlan, positions: Sequence[Position]) -> list[tuple[Position, Decimal]]:
|
||||
"""Each position and the slice of the bucket's move it carries, largest first."""
|
||||
base = sum((p.value_rub for p in positions), start=ZERO)
|
||||
if base <= ZERO:
|
||||
return []
|
||||
amount = abs(bucket.delta_value_rub)
|
||||
rows = [(p, amount * p.value_rub / base) for p in positions]
|
||||
return sorted(rows, key=lambda row: (-row[1], row[0].instrument_id))
|
||||
|
||||
|
||||
def _sell_side(bucket: BucketPlan, positions: Sequence[Position], notes: list[str]) -> list[Trade]:
|
||||
rows = _shares(bucket, positions)
|
||||
if not rows:
|
||||
if bucket.bucket != CASH:
|
||||
notes.append(
|
||||
f"бакет «{bucket.bucket}» надо уменьшить, но продавать нечего: "
|
||||
f"ни одной позиции с ценой"
|
||||
)
|
||||
return []
|
||||
out: list[Trade] = []
|
||||
for position, amount in rows:
|
||||
want = _floor_lots(amount / position.unit_value_rub, position.lot)
|
||||
# never past what is held, and the cap itself is whole lots: no accidental short
|
||||
qty = min(want, _floor_lots(position.qty, position.lot))
|
||||
if qty <= ZERO:
|
||||
continue
|
||||
out.append(
|
||||
Trade(
|
||||
instrument_id=position.instrument_id,
|
||||
ticker=position.ticker,
|
||||
name=position.name,
|
||||
action="sell",
|
||||
qty=qty,
|
||||
lot=position.lot,
|
||||
price=position.price,
|
||||
price_currency=position.price_currency,
|
||||
amount_rub=qty * position.unit_value_rub,
|
||||
current_value_rub=position.value_rub,
|
||||
current_weight=bucket.current_weight,
|
||||
)
|
||||
)
|
||||
if not out:
|
||||
notes.append(
|
||||
f"бакет «{bucket.bucket}» перевешен, но продажа меньше одного лота — рекомендации нет"
|
||||
)
|
||||
return out
|
||||
|
||||
|
||||
def _buy_side(
|
||||
bucket: BucketPlan, positions: Sequence[Position], cash: Decimal, notes: list[str]
|
||||
) -> tuple[list[Trade], Decimal]:
|
||||
rows = _shares(bucket, positions)
|
||||
if not rows:
|
||||
if bucket.bucket != CASH:
|
||||
notes.append(
|
||||
f"бакет «{bucket.bucket}» надо увеличить, но покупать нечего: "
|
||||
f"ни одной позиции с ценой"
|
||||
)
|
||||
return [], cash
|
||||
out: list[Trade] = []
|
||||
left = cash
|
||||
for position, amount in rows:
|
||||
want = _floor_lots(amount / position.unit_value_rub, position.lot)
|
||||
if want <= ZERO:
|
||||
continue
|
||||
affordable = _floor_lots(left / position.unit_value_rub, position.lot)
|
||||
blocked = affordable < want
|
||||
qty = min(want, affordable)
|
||||
if qty <= ZERO and not blocked:
|
||||
continue
|
||||
cost = qty * position.unit_value_rub
|
||||
left -= cost
|
||||
out.append(
|
||||
Trade(
|
||||
instrument_id=position.instrument_id,
|
||||
ticker=position.ticker,
|
||||
name=position.name,
|
||||
action="buy",
|
||||
qty=qty,
|
||||
lot=position.lot,
|
||||
price=position.price,
|
||||
price_currency=position.price_currency,
|
||||
amount_rub=cost,
|
||||
blocked_by_cash=blocked,
|
||||
current_value_rub=position.value_rub,
|
||||
current_weight=bucket.current_weight,
|
||||
)
|
||||
)
|
||||
if any(t.blocked_by_cash for t in out):
|
||||
notes.append(f"на докупку бакета «{bucket.bucket}» не хватает свободных денег")
|
||||
return out, left
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- loading
|
||||
|
||||
|
||||
async def _load_positions(
|
||||
session: AsyncSession, scope: str, dimension: AllocationDimension
|
||||
) -> tuple[dict[str, list[Position]], list[str]]:
|
||||
"""Priced long positions of the scope, grouped into the dimension's buckets."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
MetricHolding.instrument_id,
|
||||
MetricHolding.qty,
|
||||
MetricHolding.value_rub,
|
||||
MetricHolding.accrued_interest_rub,
|
||||
MetricHolding.market_price,
|
||||
MetricHolding.price_currency,
|
||||
Instrument.ticker,
|
||||
Instrument.name,
|
||||
Instrument.lot,
|
||||
Instrument.asset_class,
|
||||
Instrument.sector,
|
||||
Instrument.country,
|
||||
Instrument.currency,
|
||||
)
|
||||
.join(Instrument, Instrument.id == MetricHolding.instrument_id)
|
||||
.where(MetricHolding.scope == scope)
|
||||
)
|
||||
).all()
|
||||
|
||||
out: dict[str, list[Position]] = defaultdict(list)
|
||||
unpriced: list[str] = []
|
||||
for r in rows:
|
||||
qty = Decimal(r.qty or 0)
|
||||
if qty <= ZERO:
|
||||
continue
|
||||
if r.value_rub is None or r.market_price is None:
|
||||
unpriced.append(r.ticker)
|
||||
continue
|
||||
clean = Decimal(r.value_rub) - Decimal(r.accrued_interest_rub or 0)
|
||||
unit = clean / qty
|
||||
if unit <= ZERO:
|
||||
unpriced.append(r.ticker)
|
||||
continue
|
||||
bucket = bucket_of(
|
||||
Holding(
|
||||
instrument_id=r.instrument_id,
|
||||
value_rub=clean,
|
||||
asset_class=str(r.asset_class),
|
||||
sector=r.sector,
|
||||
country=r.country,
|
||||
currency=r.currency,
|
||||
),
|
||||
dimension,
|
||||
)
|
||||
out[bucket].append(
|
||||
Position(
|
||||
instrument_id=r.instrument_id,
|
||||
ticker=r.ticker,
|
||||
name=r.name,
|
||||
lot=max(1, int(r.lot or 1)),
|
||||
qty=qty,
|
||||
unit_value_rub=unit,
|
||||
price=Decimal(r.market_price),
|
||||
price_currency=r.price_currency or r.currency,
|
||||
)
|
||||
)
|
||||
return dict(out), sorted(set(unpriced))
|
||||
|
||||
|
||||
async def _load_buckets(
|
||||
session: AsyncSession, scope: str, dimension: AllocationDimension
|
||||
) -> dict[str, Decimal]:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(MetricAllocation.bucket, MetricAllocation.value_rub).where(
|
||||
MetricAllocation.scope == scope, MetricAllocation.dimension == dimension
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {r.bucket: Decimal(r.value_rub) for r in rows}
|
||||
|
||||
|
||||
async def load_targets(
|
||||
session: AsyncSession, portfolio_id: int, dimension: AllocationDimension
|
||||
) -> dict[str, Target]:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(PortfolioTarget).where(
|
||||
PortfolioTarget.portfolio_id == portfolio_id,
|
||||
PortfolioTarget.dimension == dimension,
|
||||
)
|
||||
)
|
||||
).scalars()
|
||||
return {
|
||||
r.bucket: Target(weight=Decimal(r.target_weight), band=Decimal(r.band or 0)) for r in rows
|
||||
}
|
||||
|
||||
|
||||
async def portfolio_cash_rub(session: AsyncSession, portfolio_id: int) -> Decimal:
|
||||
"""Free money on the portfolio's accounts, converted to RUB at today's rate.
|
||||
|
||||
A currency with no rate today is left out rather than counted at a made-up rate; the
|
||||
result is a purchasing power that is understated, never overstated, which is the safe
|
||||
direction for something that caps a buy.
|
||||
"""
|
||||
account_ids = set(
|
||||
(
|
||||
await session.execute(
|
||||
select(PortfolioAccount.account_id).where(
|
||||
PortfolioAccount.portfolio_id == portfolio_id
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
if not account_ids:
|
||||
return ZERO
|
||||
balances = await cash_balances(session)
|
||||
if not balances:
|
||||
return ZERO
|
||||
fx = await FxTable.load(session)
|
||||
as_of = today_local()
|
||||
total = ZERO
|
||||
for (account_id, ccy), amount in balances.items():
|
||||
if account_id not in account_ids or amount == ZERO:
|
||||
continue
|
||||
rub = fx.to_rub(amount, ccy, as_of)
|
||||
if rub is not None:
|
||||
total += rub
|
||||
return total
|
||||
|
||||
|
||||
async def compute_rebalance(
|
||||
session: AsyncSession,
|
||||
portfolio_id: int,
|
||||
dimension: AllocationDimension,
|
||||
*,
|
||||
cash_available_rub: Decimal | None = None,
|
||||
) -> RebalancePlan:
|
||||
"""The plan for one portfolio and dimension, computed from the current metric tables.
|
||||
|
||||
Used by both the refresh step and the API, so a what-if with a different cash figure
|
||||
cannot drift away from what the stored table says.
|
||||
"""
|
||||
scope = f"portfolio:{portfolio_id}"
|
||||
targets = await load_targets(session, portfolio_id, dimension)
|
||||
bucket_values = await _load_buckets(session, scope, dimension)
|
||||
positions, unpriced = await _load_positions(session, scope, dimension)
|
||||
cash = (
|
||||
cash_available_rub
|
||||
if cash_available_rub is not None
|
||||
else await portfolio_cash_rub(session, portfolio_id)
|
||||
)
|
||||
total = sum((v for v in bucket_values.values() if v > ZERO), start=ZERO)
|
||||
|
||||
warnings: list[str] = []
|
||||
if unpriced:
|
||||
warnings.append(
|
||||
f"у {len(unpriced)} инструментов нет цены — в рекомендации не вошли: "
|
||||
+ ", ".join(unpriced)
|
||||
)
|
||||
return build_plan(
|
||||
portfolio_id=portfolio_id,
|
||||
dimension=dimension,
|
||||
as_of=today_local(),
|
||||
total_value_rub=total,
|
||||
bucket_values=bucket_values,
|
||||
positions=positions,
|
||||
targets=targets,
|
||||
cash_available_rub=cash,
|
||||
warnings=warnings,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- refresh step
|
||||
|
||||
|
||||
async def rebuild_rebalance(session: AsyncSession) -> None:
|
||||
"""Replace `metric_rebalance` and fill the target columns of `metric_allocation`.
|
||||
|
||||
Only portfolios that actually have targets are touched: without a decision there is
|
||||
nothing to compare against, and an empty table is the honest representation of that.
|
||||
"""
|
||||
await session.execute(delete(MetricRebalance))
|
||||
await session.execute(update(MetricAllocation).values(target_weight=None, drift=None))
|
||||
|
||||
pairs = (
|
||||
await session.execute(
|
||||
select(PortfolioTarget.portfolio_id, PortfolioTarget.dimension).distinct()
|
||||
)
|
||||
).all()
|
||||
if not pairs:
|
||||
return
|
||||
|
||||
rows: list[dict[str, object]] = []
|
||||
for portfolio_id, dimension in sorted(pairs, key=lambda p: (p[0], str(p[1]))):
|
||||
plan = await compute_rebalance(session, portfolio_id, dimension)
|
||||
scope = f"portfolio:{portfolio_id}"
|
||||
for bucket in plan.buckets:
|
||||
if bucket.target_weight is None:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"portfolio_id": portfolio_id,
|
||||
"dimension": dimension,
|
||||
"bucket": bucket.bucket,
|
||||
"instrument_id": None,
|
||||
"current_value_rub": bucket.current_value_rub,
|
||||
"current_weight": bucket.current_weight,
|
||||
"target_weight": bucket.target_weight,
|
||||
"delta_value_rub": bucket.delta_value_rub,
|
||||
"suggested_qty": None,
|
||||
"lot": None,
|
||||
"price": None,
|
||||
"price_currency": None,
|
||||
"within_band": bucket.within_band,
|
||||
"blocked_by_cash": False,
|
||||
}
|
||||
)
|
||||
await session.execute(
|
||||
update(MetricAllocation)
|
||||
.where(
|
||||
MetricAllocation.scope == scope,
|
||||
MetricAllocation.dimension == dimension,
|
||||
MetricAllocation.bucket == bucket.bucket,
|
||||
)
|
||||
.values(target_weight=bucket.target_weight, drift=bucket.drift)
|
||||
)
|
||||
for trade in bucket.trades:
|
||||
signed = trade.amount_rub if trade.action == "buy" else -trade.amount_rub
|
||||
rows.append(
|
||||
{
|
||||
"portfolio_id": portfolio_id,
|
||||
"dimension": dimension,
|
||||
"bucket": bucket.bucket,
|
||||
"instrument_id": trade.instrument_id,
|
||||
"current_value_rub": trade.current_value_rub,
|
||||
"current_weight": trade.current_weight,
|
||||
"target_weight": None,
|
||||
"delta_value_rub": signed,
|
||||
"suggested_qty": trade.qty,
|
||||
"lot": trade.lot,
|
||||
"price": trade.price,
|
||||
"price_currency": trade.price_currency,
|
||||
"within_band": bucket.within_band,
|
||||
"blocked_by_cash": trade.blocked_by_cash,
|
||||
}
|
||||
)
|
||||
for note in plan.warnings:
|
||||
FINDINGS.add(
|
||||
"rebalance_incomplete",
|
||||
"warn",
|
||||
f"Портфель {portfolio_id}, разрез {dimension.value}: {note}",
|
||||
ref={"portfolio_id": portfolio_id, "dimension": dimension.value},
|
||||
)
|
||||
|
||||
if rows:
|
||||
await session.execute(insert(MetricRebalance), rows)
|
||||
log.info("rebalance: %s portfolio/dimension pairs, %s rows", len(pairs), len(rows))
|
||||
@@ -0,0 +1,402 @@
|
||||
"""The tax picture per calendar year and account — an estimate, labelled one (plan §3, §4).
|
||||
|
||||
**This module never produces an authoritative number and must never be presented as one.**
|
||||
The broker is the tax agent: it computes, withholds and files. What this exists for is two
|
||||
things the broker's certificate cannot do — let the certificate be *checked* against the
|
||||
ledger the app already has, and show, before a sale happens, what selling a position ahead
|
||||
of the three-year mark would cost.
|
||||
|
||||
What goes into the numbers, and why:
|
||||
|
||||
* **Dividends and coupons** are taken gross. `event.amount` is the cash that actually landed
|
||||
and `event.tax` is what the broker held back on the way (ledger.py: both are stored so
|
||||
that `amount + tax` is the gross), so the pair reconstructs the payment without a second
|
||||
source. Standalone `tax` events are deliberately NOT added on top: for every feed the
|
||||
project reads, the withholding is reported inside the payment operation, and counting both
|
||||
would double it. If a broker ever reports it separately, that shows up as a mismatch
|
||||
against the certificate — which is the whole point of the screen.
|
||||
* **Realised results** come from `lot_disposal.realized_pnl_rub`, which `ledger/rebuild.py`
|
||||
computed as `proceeds_rub - cost_rub` with **each leg converted at the CBR rate of its own
|
||||
date** — the purchase at the purchase date, the sale at the sale date. That is not a
|
||||
rounding choice, it is the law: for a foreign-currency security the rouble revaluation is
|
||||
part of the base (plan §7, вопрос 4), so a paper that did not move in dollars still
|
||||
realises a rouble gain when the dollar rose. The exception the plan names — Ministry of
|
||||
Finance eurobonds, where revaluation is excluded — is **not** implemented, because nothing
|
||||
in the instrument master distinguishes one reliably (`issuer` is free text and mostly
|
||||
empty). Instead, every foreign-currency disposal is counted and reported as a finding, so
|
||||
a eurobond among them can be spotted and corrected by hand rather than being silently
|
||||
mis-taxed in either direction.
|
||||
* **A disposal with no rate on one of its legs has `realized_pnl_rub = NULL`** and is left
|
||||
out of the totals entirely. It is never substituted with the other leg's rate or with
|
||||
today's: the count goes to `FINDINGS`, so the year is visibly incomplete instead of quietly
|
||||
wrong.
|
||||
* **ЛДВ (art. 219.1)** — three full years on an exchange-traded instrument. The flag is
|
||||
`lot_disposal.ldv_eligible`, produced by the single rule in `ledger/lots.py`; this module
|
||||
reads it and does not restate it. It is an estimate for the reason the plan records as
|
||||
вопрос 3: the classic three-year rule is applied to MOEX-traded papers only, and the 2025
|
||||
changes around foreign issuers and ИИС-3 are out of scope. An ИИС is an ordinary `account`
|
||||
here, so its own regime is not modelled at all.
|
||||
* **The rate is stored, not baked in.** `TAX_RATE` is a constant of this module, but it is
|
||||
written into every `metric_tax_year` row, so a year computed under one rate stays readable
|
||||
after the constant changes, and a changed rate is visible rather than invisible.
|
||||
|
||||
The base follows the shape the phase-4 contract fixes:
|
||||
|
||||
taxable_base = max(0, realized_gain + realized_loss - ldv_exempt)
|
||||
|
||||
Gains and losses are both over all disposals of the year (the loss is negative), and
|
||||
`ldv_exempt` is the gain on the LDV-eligible ones, subtracted back out. Dividends and coupons
|
||||
are outside the base: the agent withholds on them at source, which is what
|
||||
`tax_withheld_rub` records.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date, timedelta
|
||||
from decimal import ROUND_HALF_UP, Decimal
|
||||
|
||||
from sqlalchemy import delete, insert, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS, today_local
|
||||
from fintracker.ledger.lots import LDV_DAYS
|
||||
from fintracker.ledger.rebuild import EXCHANGE_CLASSES
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
Event,
|
||||
EventKind,
|
||||
EventStatus,
|
||||
Instrument,
|
||||
Lot,
|
||||
LotDisposal,
|
||||
MetricTaxYear,
|
||||
)
|
||||
from fintracker.pricing.fx import FxTable
|
||||
from fintracker.pricing.prices import PriceTable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
ZERO = Decimal(0)
|
||||
MONEY_PLACES = Decimal("0.01")
|
||||
|
||||
TAX_RATE = Decimal("0.13")
|
||||
"""НДФЛ on investment income for a resident. Written into every row, never assumed by a
|
||||
reader of the row: see the module docstring."""
|
||||
|
||||
DISCLAIMER = "Оценка. Налоговый агент — брокер; сверяйтесь с его справкой."
|
||||
"""Shown next to every number this module produces, on every screen, without exception."""
|
||||
|
||||
INCOME_KINDS = (EventKind.dividend, EventKind.coupon)
|
||||
|
||||
RUB = "RUB"
|
||||
|
||||
|
||||
def money(value: Decimal) -> Decimal:
|
||||
return value.quantize(MONEY_PLACES, rounding=ROUND_HALF_UP)
|
||||
|
||||
|
||||
def tax_on(base: Decimal, rate: Decimal = TAX_RATE) -> Decimal:
|
||||
"""Tax on a base, floored at zero — a negative base is a loss, not a refund."""
|
||||
return money(base * rate) if base > ZERO else money(ZERO)
|
||||
|
||||
|
||||
def ldv_date_for(open_date: date) -> date:
|
||||
"""First day a sale of this lot qualifies for the long-term exemption."""
|
||||
return open_date + timedelta(days=LDV_DAYS)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# refresh step
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Bucket:
|
||||
"""One (year, account) cell while it is being filled."""
|
||||
|
||||
dividends: Decimal = ZERO
|
||||
coupons: Decimal = ZERO
|
||||
withheld: Decimal = ZERO
|
||||
gain: Decimal = ZERO
|
||||
loss: Decimal = ZERO
|
||||
ldv_exempt: Decimal = ZERO
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Gaps:
|
||||
"""What had to be left out, so the totals can say so instead of pretending."""
|
||||
|
||||
income_no_fx: int = 0
|
||||
disposals_no_fx: int = 0
|
||||
foreign_disposals: int = 0
|
||||
foreign_currencies: set[str] = field(default_factory=set)
|
||||
|
||||
|
||||
async def rebuild_tax_year(session: AsyncSession) -> None:
|
||||
"""Replace `metric_tax_year` for every year and account the ledger touches."""
|
||||
await session.execute(delete(MetricTaxYear))
|
||||
|
||||
fx = await FxTable.load(session)
|
||||
buckets: dict[tuple[int, int], _Bucket] = defaultdict(_Bucket)
|
||||
gaps = _Gaps()
|
||||
|
||||
await _add_income(session, fx, buckets, gaps)
|
||||
await _add_realized(session, buckets, gaps)
|
||||
|
||||
rows = []
|
||||
for (year, account_id), b in sorted(buckets.items()):
|
||||
base = b.gain + b.loss - b.ldv_exempt
|
||||
rows.append(
|
||||
{
|
||||
"year": year,
|
||||
"account_id": account_id,
|
||||
"dividends_gross_rub": money(b.dividends),
|
||||
"coupons_gross_rub": money(b.coupons),
|
||||
"tax_withheld_rub": money(b.withheld),
|
||||
"realized_gain_rub": money(b.gain),
|
||||
"realized_loss_rub": money(b.loss),
|
||||
"ldv_exempt_rub": money(b.ldv_exempt),
|
||||
"taxable_base_rub": money(base if base > ZERO else ZERO),
|
||||
"estimated_tax_rub": tax_on(base),
|
||||
"tax_rate": TAX_RATE,
|
||||
}
|
||||
)
|
||||
if rows:
|
||||
await session.execute(insert(MetricTaxYear), rows)
|
||||
_report(gaps)
|
||||
log.info("tax: %s (year, account) rows", len(rows))
|
||||
|
||||
|
||||
async def _add_income(
|
||||
session: AsyncSession,
|
||||
fx: FxTable,
|
||||
buckets: dict[tuple[int, int], _Bucket],
|
||||
gaps: _Gaps,
|
||||
) -> None:
|
||||
"""Dividends and coupons, gross, with the tax the broker already took."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Event.account_id,
|
||||
Event.trade_date,
|
||||
Event.kind,
|
||||
Event.amount,
|
||||
Event.currency,
|
||||
Event.tax,
|
||||
Event.tax_currency,
|
||||
).where(Event.status == EventStatus.confirmed, Event.kind.in_(INCOME_KINDS))
|
||||
)
|
||||
).all()
|
||||
for r in rows:
|
||||
withheld = Decimal(r.tax or 0)
|
||||
gross = Decimal(r.amount or 0) + withheld
|
||||
gross_rub = fx.to_rub(gross, r.currency, r.trade_date)
|
||||
withheld_rub = fx.to_rub(withheld, r.tax_currency or r.currency, r.trade_date)
|
||||
if gross_rub is None or withheld_rub is None:
|
||||
gaps.income_no_fx += 1
|
||||
continue
|
||||
bucket = buckets[(r.trade_date.year, r.account_id)]
|
||||
if r.kind == EventKind.dividend:
|
||||
bucket.dividends += gross_rub
|
||||
else:
|
||||
bucket.coupons += gross_rub
|
||||
bucket.withheld += withheld_rub
|
||||
|
||||
|
||||
async def _add_realized(
|
||||
session: AsyncSession, buckets: dict[tuple[int, int], _Bucket], gaps: _Gaps
|
||||
) -> None:
|
||||
"""FIFO results, already in roubles at each leg's own date (`ledger/rebuild.py`)."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Lot.account_id,
|
||||
Lot.cost_currency,
|
||||
LotDisposal.close_date,
|
||||
LotDisposal.proceeds_currency,
|
||||
LotDisposal.realized_pnl_rub,
|
||||
LotDisposal.ldv_eligible,
|
||||
).join(Lot, Lot.id == LotDisposal.lot_id)
|
||||
)
|
||||
).all()
|
||||
for r in rows:
|
||||
if r.realized_pnl_rub is None:
|
||||
gaps.disposals_no_fx += 1
|
||||
continue
|
||||
for ccy in (r.cost_currency, r.proceeds_currency):
|
||||
if ccy and ccy.upper() != RUB:
|
||||
gaps.foreign_disposals += 1
|
||||
gaps.foreign_currencies.add(ccy.upper())
|
||||
break
|
||||
pnl = Decimal(r.realized_pnl_rub)
|
||||
bucket = buckets[(r.close_date.year, r.account_id)]
|
||||
if pnl >= ZERO:
|
||||
bucket.gain += pnl
|
||||
if r.ldv_eligible:
|
||||
bucket.ldv_exempt += pnl
|
||||
else:
|
||||
bucket.loss += pnl
|
||||
if r.ldv_eligible:
|
||||
# a loss on an exempt lot is not deductible either — art. 219.1 removes the
|
||||
# whole result from the base, not only the profitable half
|
||||
bucket.ldv_exempt += pnl
|
||||
|
||||
|
||||
def _report(gaps: _Gaps) -> None:
|
||||
if gaps.income_no_fx:
|
||||
FINDINGS.add(
|
||||
"tax_income_no_fx",
|
||||
"warn",
|
||||
f"{gaps.income_no_fx} выплат без курса ЦБ на дату — в налоговый год не вошли",
|
||||
count=gaps.income_no_fx,
|
||||
)
|
||||
if gaps.disposals_no_fx:
|
||||
FINDINGS.add(
|
||||
"tax_disposal_no_fx",
|
||||
"warn",
|
||||
f"{gaps.disposals_no_fx} закрытий лотов без курса на одну из ног — "
|
||||
"рублёвый результат неизвестен, в базу не вошли",
|
||||
count=gaps.disposals_no_fx,
|
||||
)
|
||||
if gaps.foreign_disposals:
|
||||
FINDINGS.add(
|
||||
"tax_currency_revaluation",
|
||||
"info",
|
||||
f"{gaps.foreign_disposals} закрытий в валюте "
|
||||
f"({', '.join(sorted(gaps.foreign_currencies))}): валютная переоценка включена "
|
||||
"в базу. Еврооблигации Минфина, где она не облагается, автоматически не "
|
||||
"распознаются — проверьте вручную",
|
||||
count=gaps.foreign_disposals,
|
||||
)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------------------
|
||||
# open lots: the screen that shows the cost of selling early
|
||||
# --------------------------------------------------------------------------------------
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class OpenLotTax:
|
||||
"""One open lot and what selling it today would cost — all of it an estimate."""
|
||||
|
||||
lot_id: int
|
||||
instrument_id: int
|
||||
ticker: str | None
|
||||
name: str
|
||||
account_id: int
|
||||
account_name: str
|
||||
open_date: date
|
||||
qty_remaining: Decimal
|
||||
cost_rub: Decimal | None
|
||||
market_value_rub: Decimal | None
|
||||
unrealized_gain_rub: Decimal | None
|
||||
ldv_eligible: bool
|
||||
ldv_date: date | None
|
||||
days_to_ldv: int | None
|
||||
tax_if_sold_now_rub: Decimal | None
|
||||
|
||||
|
||||
async def open_lot_tax(
|
||||
session: AsyncSession,
|
||||
*,
|
||||
as_of: date | None = None,
|
||||
account_id: int | None = None,
|
||||
rate: Decimal = TAX_RATE,
|
||||
) -> list[OpenLotTax]:
|
||||
"""Every long lot still open, with its ЛДВ date and the tax a sale today would trigger.
|
||||
|
||||
Read live rather than from a metric table: it depends on today's date and today's price,
|
||||
and a table refreshed nightly would show yesterday's `days_to_ldv` — off by one on
|
||||
exactly the day the answer matters.
|
||||
|
||||
Short lots are excluded. `qty_remaining` is stored signed, and a short position has no
|
||||
holding period to accrue: art. 219.1 is about owning a paper for three years.
|
||||
|
||||
`cost_rub` is the lot's cost at the rate of ITS OWN open date, prorated to the part still
|
||||
held — the same convention the realised numbers use, so an open lot and the closure it
|
||||
later becomes are measured the same way. `market_value_rub` is at today's rate, which is
|
||||
what makes the difference a currency-revaluation figure rather than a price-only one.
|
||||
"""
|
||||
as_of = as_of or today_local()
|
||||
conditions = [Lot.qty_remaining > 0]
|
||||
if account_id is not None:
|
||||
conditions.append(Lot.account_id == account_id)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Lot.id,
|
||||
Lot.account_id,
|
||||
Lot.instrument_id,
|
||||
Lot.open_date,
|
||||
Lot.qty_open,
|
||||
Lot.qty_remaining,
|
||||
Lot.cost_total_rub,
|
||||
Account.name.label("account_name"),
|
||||
Instrument.ticker,
|
||||
Instrument.name.label("instrument_name"),
|
||||
Instrument.asset_class,
|
||||
Instrument.board,
|
||||
)
|
||||
.join(Account, Account.id == Lot.account_id)
|
||||
.join(Instrument, Instrument.id == Lot.instrument_id)
|
||||
.where(*conditions)
|
||||
.order_by(Lot.open_date, Lot.id)
|
||||
)
|
||||
).all()
|
||||
|
||||
fx = await FxTable.load(session)
|
||||
prices = await PriceTable.load(session)
|
||||
|
||||
out: list[OpenLotTax] = []
|
||||
for r in rows:
|
||||
qty = Decimal(r.qty_remaining)
|
||||
cost_rub = None
|
||||
if r.cost_total_rub is not None and r.qty_open:
|
||||
cost_rub = money(Decimal(r.cost_total_rub) * qty / Decimal(r.qty_open))
|
||||
|
||||
quote = prices.latest(r.instrument_id, as_of)
|
||||
value_rub = None
|
||||
if quote is not None:
|
||||
value_rub = fx.to_rub(qty * quote.total, quote.currency, as_of)
|
||||
if value_rub is not None:
|
||||
value_rub = money(value_rub)
|
||||
|
||||
unrealized = None
|
||||
if cost_rub is not None and value_rub is not None:
|
||||
unrealized = money(value_rub - cost_rub)
|
||||
|
||||
exchange_traded = str(r.asset_class) in EXCHANGE_CLASSES and bool(r.board)
|
||||
ldv_date = ldv_date_for(r.open_date) if exchange_traded else None
|
||||
eligible = ldv_date is not None and as_of >= ldv_date
|
||||
days_to_ldv = None if ldv_date is None else max((ldv_date - as_of).days, 0)
|
||||
|
||||
if eligible:
|
||||
tax_now: Decimal | None = money(ZERO)
|
||||
elif unrealized is None:
|
||||
tax_now = None
|
||||
else:
|
||||
tax_now = tax_on(unrealized, rate)
|
||||
|
||||
out.append(
|
||||
OpenLotTax(
|
||||
lot_id=r.id,
|
||||
instrument_id=r.instrument_id,
|
||||
ticker=r.ticker,
|
||||
name=r.instrument_name,
|
||||
account_id=r.account_id,
|
||||
account_name=r.account_name,
|
||||
open_date=r.open_date,
|
||||
qty_remaining=qty,
|
||||
cost_rub=cost_rub,
|
||||
market_value_rub=value_rub,
|
||||
unrealized_gain_rub=unrealized,
|
||||
ldv_eligible=eligible,
|
||||
ldv_date=ldv_date,
|
||||
days_to_ldv=days_to_ldv,
|
||||
tax_if_sold_now_rub=tax_now,
|
||||
)
|
||||
)
|
||||
return out
|
||||
@@ -18,17 +18,22 @@ from fintracker.api.routers import (
|
||||
accounts,
|
||||
analytics,
|
||||
auth,
|
||||
benchmarks,
|
||||
cashflow,
|
||||
categories,
|
||||
events,
|
||||
goals,
|
||||
health,
|
||||
imports,
|
||||
income,
|
||||
instruments,
|
||||
links,
|
||||
metrics,
|
||||
networth,
|
||||
rebalance,
|
||||
rules,
|
||||
sync,
|
||||
tax,
|
||||
transactions,
|
||||
)
|
||||
from fintracker.api.web import mount_web
|
||||
@@ -95,6 +100,12 @@ def create_app() -> FastAPI:
|
||||
app.include_router(instruments.router, prefix=API_PREFIX)
|
||||
app.include_router(links.router, prefix=API_PREFIX)
|
||||
app.include_router(metrics.router, prefix=API_PREFIX)
|
||||
app.include_router(goals.router, prefix=API_PREFIX)
|
||||
app.include_router(income.router, prefix=API_PREFIX)
|
||||
app.include_router(rebalance.router, prefix=API_PREFIX)
|
||||
app.include_router(tax.router, prefix=API_PREFIX)
|
||||
app.include_router(benchmarks.router, prefix=API_PREFIX)
|
||||
app.include_router(benchmarks.analytics_router, prefix=API_PREFIX)
|
||||
if settings.web_dir is not None:
|
||||
mount_web(app, settings.web_dir, API_PREFIX)
|
||||
return app
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Benchmarks: the list the user maintains, and the comparison built from it.
|
||||
|
||||
Two routers live here because the two halves belong to two prefixes the contract fixes:
|
||||
`router` serves `/benchmarks` (CRUD on the user's choice of indices) and `analytics_router`
|
||||
serves `/analytics/benchmarks` (the read of `metric_benchmark_returns` beside
|
||||
`metric_returns`). Both are exported for `api/app.py` to include.
|
||||
|
||||
The comparison itself is not computed here — `analytics/benchmarks.py` built it during the
|
||||
refresh, on the portfolio's own date grid. Only `excess` is derived per request, as a
|
||||
subtraction of two numbers that are already on the same grid.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Query, Response, status
|
||||
from sqlalchemy import delete, func, select
|
||||
|
||||
from fintracker.api.deps import CurrentUser, SessionDep
|
||||
from fintracker.api.errors import Problem
|
||||
from fintracker.api.schemas.benchmarks import (
|
||||
BenchmarkComparison,
|
||||
BenchmarkComparisonOut,
|
||||
BenchmarkCreate,
|
||||
BenchmarkOut,
|
||||
BenchmarkPatch,
|
||||
BenchmarkReturnOut,
|
||||
)
|
||||
from fintracker.api.scopes import DEFAULT_SCOPE, resolve_scope
|
||||
from fintracker.models import (
|
||||
Benchmark,
|
||||
BenchmarkKind,
|
||||
MetricBenchmarkReturns,
|
||||
MetricReturns,
|
||||
PriceDaily,
|
||||
)
|
||||
|
||||
router = APIRouter(prefix="/benchmarks", tags=["benchmarks"])
|
||||
analytics_router = APIRouter(prefix="/analytics", tags=["benchmarks"])
|
||||
|
||||
#: Order the periods are shown in; the tables store them unordered.
|
||||
PERIOD_ORDER = {p: i for i, p in enumerate(("1m", "3m", "6m", "ytd", "1y", "3y", "all"))}
|
||||
|
||||
ScopeParam = Annotated[str, Query(description="all | account:<id> | portfolio:<id>")]
|
||||
|
||||
|
||||
def _kind(value: str) -> BenchmarkKind:
|
||||
try:
|
||||
return BenchmarkKind(value)
|
||||
except ValueError:
|
||||
allowed = ", ".join(k.value for k in BenchmarkKind)
|
||||
raise Problem(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY, "Unprocessable", f"kind: ожидается {allowed}"
|
||||
) from None
|
||||
|
||||
|
||||
async def _history(session: SessionDep) -> dict[int, tuple[date, date]]:
|
||||
"""The price history each benchmark instrument actually has, for the list response."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
PriceDaily.instrument_id, func.min(PriceDaily.d), func.max(PriceDaily.d)
|
||||
).group_by(PriceDaily.instrument_id)
|
||||
)
|
||||
).all()
|
||||
return {r[0]: (r[1], r[2]) for r in rows}
|
||||
|
||||
|
||||
def _out(benchmark: Benchmark, history: dict[int, tuple[date, date]]) -> BenchmarkOut:
|
||||
span = history.get(benchmark.instrument_id or 0)
|
||||
return BenchmarkOut(
|
||||
id=benchmark.id,
|
||||
code=benchmark.code,
|
||||
name=benchmark.name,
|
||||
kind=str(benchmark.kind),
|
||||
source=benchmark.source,
|
||||
currency=benchmark.currency,
|
||||
is_default=benchmark.is_default,
|
||||
is_active=benchmark.is_active,
|
||||
instrument_id=benchmark.instrument_id,
|
||||
history_from=span[0] if span else None,
|
||||
history_to=span[1] if span else None,
|
||||
)
|
||||
|
||||
|
||||
@router.get("", name="list")
|
||||
async def list_benchmarks(session: SessionDep, _: CurrentUser) -> list[BenchmarkOut]:
|
||||
"""Every benchmark, defaults first — the order the comparison block shows them in."""
|
||||
found = await session.execute(
|
||||
select(Benchmark).order_by(Benchmark.is_default.desc(), Benchmark.code)
|
||||
)
|
||||
history = await _history(session)
|
||||
return [_out(b, history) for b in found.scalars()]
|
||||
|
||||
|
||||
@router.post("", name="create", status_code=status.HTTP_201_CREATED)
|
||||
async def create_benchmark(
|
||||
body: BenchmarkCreate, session: SessionDep, _: CurrentUser
|
||||
) -> BenchmarkOut:
|
||||
existing = (
|
||||
await session.execute(select(Benchmark).where(Benchmark.code == body.code))
|
||||
).scalar_one_or_none()
|
||||
if existing is not None:
|
||||
raise Problem(status.HTTP_409_CONFLICT, "Conflict", f"Бенчмарк {body.code} уже есть")
|
||||
data = body.model_dump()
|
||||
data["kind"] = _kind(body.kind)
|
||||
benchmark = Benchmark(**data)
|
||||
session.add(benchmark)
|
||||
await session.commit()
|
||||
await session.refresh(benchmark)
|
||||
return _out(benchmark, await _history(session))
|
||||
|
||||
|
||||
@router.patch("/{benchmark_id}", name="patch")
|
||||
async def patch_benchmark(
|
||||
benchmark_id: int, body: BenchmarkPatch, session: SessionDep, _: CurrentUser
|
||||
) -> BenchmarkOut:
|
||||
benchmark = await session.get(Benchmark, benchmark_id)
|
||||
if benchmark is None:
|
||||
raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет бенчмарка {benchmark_id}")
|
||||
changes = body.model_dump(exclude_unset=True)
|
||||
for field in ("code", "name", "kind", "source", "currency", "is_default", "is_active"):
|
||||
if field in changes and changes[field] is None:
|
||||
raise Problem(status.HTTP_400_BAD_REQUEST, "Bad request", f"{field} не может быть null")
|
||||
if "kind" in changes:
|
||||
changes["kind"] = _kind(changes["kind"])
|
||||
for field, value in changes.items():
|
||||
setattr(benchmark, field, value)
|
||||
await session.commit()
|
||||
await session.refresh(benchmark)
|
||||
return _out(benchmark, await _history(session))
|
||||
|
||||
|
||||
@router.delete("/{benchmark_id}", name="delete", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_benchmark(benchmark_id: int, session: SessionDep, _: CurrentUser) -> Response:
|
||||
benchmark = await session.get(Benchmark, benchmark_id)
|
||||
if benchmark is None:
|
||||
raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет бенчмарка {benchmark_id}")
|
||||
# the metric rows go with it: a comparison against an index nobody tracks any more is
|
||||
# not a number the screens should still be able to find
|
||||
await session.execute(
|
||||
delete(MetricBenchmarkReturns).where(MetricBenchmarkReturns.benchmark_id == benchmark_id)
|
||||
)
|
||||
await session.delete(benchmark)
|
||||
await session.commit()
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@analytics_router.get("/benchmarks", name="benchmarks")
|
||||
async def compare(
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
scope: ScopeParam = DEFAULT_SCOPE,
|
||||
period: Annotated[
|
||||
list[str] | None, Query(description="repeatable: 1m 3m 6m ytd 1y 3y all")
|
||||
] = None,
|
||||
) -> BenchmarkComparison:
|
||||
"""The portfolio's TWR beside each benchmark's, on the same grid of days.
|
||||
|
||||
`days_skipped` is returned on both sides and is not cosmetic: a non-zero value on either
|
||||
means the two chains did not cover the same days, and the difference is then an
|
||||
approximation of an excess return rather than one.
|
||||
"""
|
||||
await resolve_scope(session, scope)
|
||||
conditions = [MetricReturns.scope == scope]
|
||||
if period:
|
||||
conditions.append(MetricReturns.period.in_(period))
|
||||
portfolio = (
|
||||
await session.execute(
|
||||
select(
|
||||
MetricReturns.period,
|
||||
MetricReturns.date_from,
|
||||
MetricReturns.date_to,
|
||||
MetricReturns.twr,
|
||||
MetricReturns.twr_annualized,
|
||||
MetricReturns.twr_days_skipped,
|
||||
).where(*conditions)
|
||||
)
|
||||
).all()
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
MetricBenchmarkReturns.period,
|
||||
MetricBenchmarkReturns.twr,
|
||||
MetricBenchmarkReturns.twr_annualized,
|
||||
MetricBenchmarkReturns.days_skipped,
|
||||
Benchmark.id,
|
||||
Benchmark.code,
|
||||
Benchmark.name,
|
||||
Benchmark.kind,
|
||||
)
|
||||
.join(Benchmark, Benchmark.id == MetricBenchmarkReturns.benchmark_id)
|
||||
.where(MetricBenchmarkReturns.scope == scope)
|
||||
.order_by(Benchmark.is_default.desc(), Benchmark.code)
|
||||
)
|
||||
).all()
|
||||
by_period: dict[str, list[BenchmarkReturnOut]] = {}
|
||||
for r in rows:
|
||||
by_period.setdefault(r.period, []).append(
|
||||
BenchmarkReturnOut(
|
||||
benchmark_id=r.id,
|
||||
code=r.code,
|
||||
name=r.name,
|
||||
kind=str(r.kind),
|
||||
twr=r.twr,
|
||||
twr_annualized=r.twr_annualized,
|
||||
days_skipped=r.days_skipped,
|
||||
excess=None,
|
||||
)
|
||||
)
|
||||
|
||||
out = []
|
||||
for p in sorted(portfolio, key=lambda r: PERIOD_ORDER.get(r.period, 99)):
|
||||
benchmarks = []
|
||||
for b in by_period.get(p.period, []):
|
||||
excess = None if p.twr is None or b.twr is None else p.twr - b.twr
|
||||
benchmarks.append(b.model_copy(update={"excess": excess}))
|
||||
out.append(
|
||||
BenchmarkComparisonOut(
|
||||
period=p.period,
|
||||
date_from=p.date_from,
|
||||
date_to=p.date_to,
|
||||
portfolio_twr=p.twr,
|
||||
portfolio_twr_annualized=p.twr_annualized,
|
||||
portfolio_days_skipped=p.twr_days_skipped,
|
||||
benchmarks=benchmarks,
|
||||
)
|
||||
)
|
||||
return BenchmarkComparison(rows=out)
|
||||
@@ -0,0 +1,113 @@
|
||||
"""Goals: CRUD plus the derived progress (docs/ai/phase4-contract.md §4).
|
||||
|
||||
Progress is computed per request through `analytics/goals.compute_goal_progress` for the
|
||||
same reason the rebalancing endpoint does: a goal created a minute ago has no row in
|
||||
`metric_goal_progress` yet, and «no data» for a goal the user just typed in reads as a bug.
|
||||
The refresh step fills the table from the identical function, so the screen and the stored
|
||||
metric cannot drift apart.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import APIRouter, Response, status
|
||||
from sqlalchemy import select
|
||||
|
||||
from fintracker.analytics.goals import compute_goal_progress
|
||||
from fintracker.api.deps import CurrentUser, SessionDep
|
||||
from fintracker.api.errors import Problem
|
||||
from fintracker.api.schemas.goals import GoalCreate, GoalOut, GoalPatch, GoalProgressOut
|
||||
from fintracker.api.scopes import resolve_scope
|
||||
from fintracker.models import Goal
|
||||
|
||||
router = APIRouter(prefix="/goals", tags=["goals"])
|
||||
|
||||
|
||||
async def _goal(session: SessionDep, goal_id: int) -> Goal:
|
||||
goal = await session.get(Goal, goal_id)
|
||||
if goal is None:
|
||||
raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет цели с id {goal_id}")
|
||||
return goal
|
||||
|
||||
|
||||
async def _check_name(session: SessionDep, name: str, *, exclude: int | None = None) -> None:
|
||||
stmt = select(Goal.id).where(Goal.name == name)
|
||||
if exclude is not None:
|
||||
stmt = stmt.where(Goal.id != exclude)
|
||||
if (await session.execute(stmt)).scalar_one_or_none() is not None:
|
||||
raise Problem(status.HTTP_409_CONFLICT, "Conflict", f"Цель «{name}» уже есть")
|
||||
|
||||
|
||||
@router.get("", name="list")
|
||||
async def list_goals(
|
||||
session: SessionDep, _: CurrentUser, include_archived: bool = False
|
||||
) -> list[GoalOut]:
|
||||
stmt = select(Goal).order_by(Goal.id)
|
||||
if not include_archived:
|
||||
stmt = stmt.where(Goal.archived.is_(False))
|
||||
rows = (await session.execute(stmt)).scalars().all()
|
||||
return [GoalOut.model_validate(g, from_attributes=True) for g in rows]
|
||||
|
||||
|
||||
@router.post("", name="create", status_code=status.HTTP_201_CREATED)
|
||||
async def create_goal(body: GoalCreate, session: SessionDep, _: CurrentUser) -> GoalOut:
|
||||
await _check_name(session, body.name)
|
||||
# a goal pointed at a scope the metrics never built would silently read as zero forever
|
||||
await resolve_scope(session, body.scope)
|
||||
goal = Goal(**body.model_dump())
|
||||
session.add(goal)
|
||||
await session.commit()
|
||||
await session.refresh(goal)
|
||||
return GoalOut.model_validate(goal, from_attributes=True)
|
||||
|
||||
|
||||
@router.patch("/{goal_id}", name="patch")
|
||||
async def patch_goal(goal_id: int, body: GoalPatch, session: SessionDep, _: CurrentUser) -> GoalOut:
|
||||
goal = await _goal(session, goal_id)
|
||||
changes = body.model_dump(exclude_unset=True)
|
||||
for field in ("name", "scope", "target_amount", "currency", "archived"):
|
||||
if field in changes and changes[field] is None:
|
||||
raise Problem(
|
||||
status.HTTP_400_BAD_REQUEST, "Bad request", f"Поле {field} не может быть null"
|
||||
)
|
||||
if "name" in changes:
|
||||
await _check_name(session, changes["name"], exclude=goal_id)
|
||||
if "scope" in changes:
|
||||
await resolve_scope(session, changes["scope"])
|
||||
for field, value in changes.items():
|
||||
setattr(goal, field, value)
|
||||
await session.commit()
|
||||
await session.refresh(goal)
|
||||
return GoalOut.model_validate(goal, from_attributes=True)
|
||||
|
||||
|
||||
@router.delete("/{goal_id}", name="delete", status_code=status.HTTP_204_NO_CONTENT)
|
||||
async def delete_goal(goal_id: int, session: SessionDep, _: CurrentUser) -> Response:
|
||||
goal = await _goal(session, goal_id)
|
||||
await session.delete(goal)
|
||||
await session.commit()
|
||||
return Response(status_code=status.HTTP_204_NO_CONTENT)
|
||||
|
||||
|
||||
@router.get("/{goal_id}/progress", name="progress")
|
||||
async def goal_progress(goal_id: int, session: SessionDep, _: CurrentUser) -> GoalProgressOut:
|
||||
goal = await _goal(session, goal_id)
|
||||
progress = await compute_goal_progress(session, goal)
|
||||
if progress is None:
|
||||
raise Problem(
|
||||
status.HTTP_409_CONFLICT,
|
||||
"Conflict",
|
||||
f"Прогресс цели «{goal.name}» не считается: нет курса для {goal.currency} "
|
||||
f"или сумма цели неположительна",
|
||||
)
|
||||
return GoalProgressOut(
|
||||
goal_id=progress.goal_id,
|
||||
as_of=progress.as_of,
|
||||
current_value_rub=progress.current_value_rub,
|
||||
target_amount_rub=progress.target_amount_rub,
|
||||
progress=progress.progress,
|
||||
projected_date=progress.projected_date,
|
||||
basis=progress.basis,
|
||||
assumed_rate=progress.assumed_rate,
|
||||
monthly_needed_rub=progress.monthly_needed_rub,
|
||||
on_track=progress.on_track,
|
||||
)
|
||||
@@ -0,0 +1,288 @@
|
||||
"""`/income` — the dividend and coupon calendar, its history and its forecast (contract §1).
|
||||
|
||||
Every endpoint is a read of `metric_income_calendar` / `metric_income_monthly`, which
|
||||
`analytics/income.py` rebuilt during the last refresh. Nothing is computed per request except
|
||||
the grouping the screen asks for, so the three tabs of the screen cannot disagree.
|
||||
|
||||
The one rule worth repeating here: **future rows are split by `basis` wherever they are
|
||||
summed.** `total_expected_rub` and each month's `amount_rub` are still handed over as one
|
||||
number because the screen needs one, but `by_basis` sits next to it every time, so a total
|
||||
that is mostly `history` can be recognised as mostly guesswork.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections import defaultdict
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from sqlalchemy import select
|
||||
|
||||
from fintracker.analytics import today_local
|
||||
from fintracker.analytics.income import add_months, month_start
|
||||
from fintracker.api.deps import CurrentUser, SessionDep
|
||||
from fintracker.api.schemas.income import (
|
||||
CalendarEntry,
|
||||
CalendarOut,
|
||||
ForecastMonth,
|
||||
ForecastOut,
|
||||
HistoryOut,
|
||||
HistoryRow,
|
||||
HistoryTotals,
|
||||
)
|
||||
from fintracker.api.scopes import DEFAULT_SCOPE, resolve_scope
|
||||
from fintracker.models import (
|
||||
IncomeBasis,
|
||||
Instrument,
|
||||
MetricDataQuality,
|
||||
MetricIncomeCalendar,
|
||||
MetricIncomeMonthly,
|
||||
MetricPortfolioValueDaily,
|
||||
)
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
router = APIRouter(prefix="/income", tags=["income"])
|
||||
|
||||
ZERO = Decimal(0)
|
||||
RUB = "RUB"
|
||||
|
||||
DEFAULT_FORECAST_MONTHS = 12
|
||||
|
||||
#: Findings `analytics/income.py` raises; the forecast repeats them so the screen can show
|
||||
#: what it could not cover instead of silently under-reporting.
|
||||
INCOME_CHECKS = ("income_without_history", "income_irregular_history", "income_missing_fx")
|
||||
|
||||
ScopeParam = Annotated[str, Query(description="all | account:<id> | portfolio:<id>")]
|
||||
|
||||
|
||||
@router.get("/calendar", name="calendar")
|
||||
async def calendar(
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
scope: ScopeParam = DEFAULT_SCOPE,
|
||||
date_from: Annotated[date | None, Query(alias="date_from")] = None,
|
||||
date_to: Annotated[date | None, Query(alias="date_to")] = None,
|
||||
include_paid: bool = False,
|
||||
) -> CalendarOut:
|
||||
"""Payments in a window, one row each; defaults to the next 12 months, forecast only."""
|
||||
await resolve_scope(session, scope)
|
||||
as_of = today_local()
|
||||
start = date_from if date_from is not None else as_of
|
||||
# the default window is the twelve months AHEAD, half-open: a quarterly payer must give
|
||||
# four entries, and an inclusive year-end boundary would let a fifth in on some dates
|
||||
end = (
|
||||
date_to
|
||||
if date_to is not None
|
||||
else add_months(as_of, DEFAULT_FORECAST_MONTHS) - timedelta(days=1)
|
||||
)
|
||||
|
||||
conditions = [
|
||||
MetricIncomeCalendar.scope == scope,
|
||||
MetricIncomeCalendar.expected_date >= start,
|
||||
MetricIncomeCalendar.expected_date <= end,
|
||||
]
|
||||
if not include_paid:
|
||||
conditions.append(MetricIncomeCalendar.basis != IncomeBasis.paid)
|
||||
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(MetricIncomeCalendar, Instrument)
|
||||
.join(Instrument, Instrument.id == MetricIncomeCalendar.instrument_id)
|
||||
.where(*conditions)
|
||||
.order_by(MetricIncomeCalendar.expected_date, Instrument.ticker)
|
||||
)
|
||||
).all()
|
||||
|
||||
entries = [_entry(row, instrument) for row, instrument in rows]
|
||||
by_basis: dict[str, Decimal] = defaultdict(lambda: ZERO)
|
||||
total = ZERO
|
||||
for row, _instrument in rows:
|
||||
if row.basis == IncomeBasis.paid or row.amount_rub is None:
|
||||
continue
|
||||
by_basis[str(row.basis.value)] += Decimal(row.amount_rub)
|
||||
total += Decimal(row.amount_rub)
|
||||
|
||||
return CalendarOut(
|
||||
as_of=as_of,
|
||||
currency=RUB,
|
||||
total_expected_rub=total,
|
||||
entries=entries,
|
||||
by_basis=dict(sorted(by_basis.items())),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/history", name="history")
|
||||
async def history(
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
scope: ScopeParam = DEFAULT_SCOPE,
|
||||
group: Annotated[str, Query(description="month")] = "month",
|
||||
date_from: Annotated[date | None, Query(alias="date_from")] = None,
|
||||
date_to: Annotated[date | None, Query(alias="date_to")] = None,
|
||||
kind: Annotated[str | None, Query(description="dividend | coupon | …")] = None,
|
||||
) -> HistoryOut:
|
||||
"""Income actually received, grouped by month, kind and currency.
|
||||
|
||||
`group` exists for the contract's sake and accepts only `month`: the table is stored
|
||||
monthly, and a finer grouping would have to re-read the ledger, which is what the
|
||||
calendar's `paid` rows already do per payment.
|
||||
"""
|
||||
await resolve_scope(session, scope)
|
||||
conditions = [MetricIncomeMonthly.scope == scope]
|
||||
if date_from is not None:
|
||||
conditions.append(MetricIncomeMonthly.month >= month_start(date_from))
|
||||
if date_to is not None:
|
||||
conditions.append(MetricIncomeMonthly.month <= date_to)
|
||||
if kind is not None:
|
||||
conditions.append(MetricIncomeMonthly.kind == kind)
|
||||
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(MetricIncomeMonthly)
|
||||
.where(*conditions)
|
||||
.order_by(
|
||||
MetricIncomeMonthly.month,
|
||||
MetricIncomeMonthly.kind,
|
||||
MetricIncomeMonthly.currency,
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
total_rub = ZERO
|
||||
total_tax_rub = ZERO
|
||||
for r in rows:
|
||||
if r.amount_rub is None:
|
||||
continue
|
||||
total_rub += Decimal(r.amount_rub)
|
||||
# the tax is stored native; it converts at the same effective rate the month's own
|
||||
# amount did, which is exact for a single-payment month and right on average otherwise
|
||||
if r.tax_withheld and r.amount:
|
||||
total_tax_rub += Decimal(r.tax_withheld) * Decimal(r.amount_rub) / Decimal(r.amount)
|
||||
|
||||
return HistoryOut(
|
||||
rows=[
|
||||
HistoryRow(
|
||||
month=r.month,
|
||||
kind=r.kind,
|
||||
currency=r.currency,
|
||||
amount=r.amount,
|
||||
amount_rub=r.amount_rub,
|
||||
tax_withheld=r.tax_withheld,
|
||||
payment_count=r.payment_count,
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
totals=HistoryTotals(amount_rub=total_rub, tax_withheld_rub=total_tax_rub),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/forecast", name="forecast")
|
||||
async def forecast(
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
scope: ScopeParam = DEFAULT_SCOPE,
|
||||
months: Annotated[int, Query(ge=1, le=36)] = DEFAULT_FORECAST_MONTHS,
|
||||
) -> ForecastOut:
|
||||
"""Expected income per month, split by basis, plus the yield it implies."""
|
||||
await resolve_scope(session, scope)
|
||||
as_of = today_local()
|
||||
end = add_months(as_of, months)
|
||||
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(MetricIncomeCalendar).where(
|
||||
MetricIncomeCalendar.scope == scope,
|
||||
MetricIncomeCalendar.basis != IncomeBasis.paid,
|
||||
MetricIncomeCalendar.expected_date >= as_of,
|
||||
# half-open, like the calendar's default window and for the same reason
|
||||
MetricIncomeCalendar.expected_date < end,
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
|
||||
buckets: dict[date, dict[str, Decimal]] = defaultdict(lambda: defaultdict(lambda: ZERO))
|
||||
total = ZERO
|
||||
for r in rows:
|
||||
if r.amount_rub is None:
|
||||
continue # unconvertible: counted in `income_missing_fx`, not silently as zero
|
||||
amount = Decimal(r.amount_rub)
|
||||
buckets[month_start(r.expected_date)][str(r.basis.value)] += amount
|
||||
total += amount
|
||||
|
||||
out_months = [
|
||||
ForecastMonth(
|
||||
month=month,
|
||||
amount_rub=sum(by_basis.values(), start=ZERO),
|
||||
by_basis=dict(sorted(by_basis.items())),
|
||||
)
|
||||
for month, by_basis in sorted(buckets.items())
|
||||
]
|
||||
|
||||
return ForecastOut(
|
||||
months=out_months,
|
||||
total_rub=total,
|
||||
annual_yield_on_value=await _yield_on_value(session, scope, total, months),
|
||||
warnings=await _warnings(session),
|
||||
)
|
||||
|
||||
|
||||
def _entry(row: MetricIncomeCalendar, instrument: Instrument) -> CalendarEntry:
|
||||
return CalendarEntry(
|
||||
instrument_id=row.instrument_id,
|
||||
ticker=instrument.ticker,
|
||||
name=instrument.name,
|
||||
kind=row.kind,
|
||||
expected_date=row.expected_date,
|
||||
record_date=row.record_date,
|
||||
qty=row.qty,
|
||||
per_unit=row.per_unit,
|
||||
amount=row.amount,
|
||||
currency=row.currency,
|
||||
amount_rub=row.amount_rub,
|
||||
basis=str(row.basis.value),
|
||||
tax_withheld=row.tax_withheld,
|
||||
)
|
||||
|
||||
|
||||
async def _yield_on_value(
|
||||
session: SessionDep, scope: str, total: Decimal, months: int
|
||||
) -> Decimal | None:
|
||||
"""Expected income annualised over the scope's latest total value; None without one."""
|
||||
value = (
|
||||
await session.execute(
|
||||
select(MetricPortfolioValueDaily.total_rub)
|
||||
.where(MetricPortfolioValueDaily.scope == scope)
|
||||
.order_by(MetricPortfolioValueDaily.d.desc())
|
||||
.limit(1)
|
||||
)
|
||||
).scalar_one_or_none()
|
||||
if value is None or Decimal(value) <= ZERO or months <= 0:
|
||||
return None
|
||||
return total * Decimal(12) / Decimal(months) / Decimal(value)
|
||||
|
||||
|
||||
async def _warnings(session: SessionDep) -> list[str]:
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(MetricDataQuality.detail)
|
||||
.where(MetricDataQuality.check_name.in_(INCOME_CHECKS))
|
||||
.order_by(MetricDataQuality.check_name)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return list(rows)
|
||||
@@ -0,0 +1,231 @@
|
||||
"""Target weights and rebalancing suggestions (docs/ai/phase4-contract.md §2).
|
||||
|
||||
Unlike the read-only `/analytics/*` endpoints, the suggestion here is computed per request
|
||||
through `analytics/rebalance.compute_rebalance` rather than read from `metric_rebalance`.
|
||||
Two reasons, and they are the same reason twice: the what-if parameter `cash_available`
|
||||
changes the answer and therefore cannot come from a stored table, and a target the user just
|
||||
saved must be visible before the next refresh — a screen that shows yesterday's targets
|
||||
next to today's prices is a screen that lies. The stored table stays the canonical copy for
|
||||
everything that reads metrics in bulk, and both come out of the same function, so they
|
||||
cannot disagree.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Query, status
|
||||
from sqlalchemy import delete, select
|
||||
|
||||
from fintracker.analytics.rebalance import DEFAULT_DIMENSION, compute_rebalance
|
||||
from fintracker.api.deps import CurrentUser, SessionDep
|
||||
from fintracker.api.errors import Problem
|
||||
from fintracker.api.schemas.rebalance import (
|
||||
RebalanceBucketOut,
|
||||
RebalanceOut,
|
||||
TargetOut,
|
||||
TargetsIn,
|
||||
TargetsOut,
|
||||
TradeOut,
|
||||
)
|
||||
from fintracker.models import AllocationDimension, Portfolio, PortfolioTarget
|
||||
|
||||
router = APIRouter(prefix="/portfolios", tags=["rebalance"])
|
||||
|
||||
ZERO = Decimal(0)
|
||||
ONE = Decimal(1)
|
||||
|
||||
WEIGHT_TOLERANCE = Decimal("0.0001")
|
||||
"""How far the weights of one dimension may add up from 1 before the set is refused.
|
||||
The server does not normalise: a total of 0.9 is a mistake in the plan, not a scale factor,
|
||||
and silently stretching it would hide the missing tenth of the portfolio forever."""
|
||||
|
||||
DimensionParam = Annotated[str, Query(description="asset_class | sector | country | currency")]
|
||||
|
||||
|
||||
def parse_dimension(value: str) -> AllocationDimension:
|
||||
try:
|
||||
return AllocationDimension(value)
|
||||
except ValueError:
|
||||
allowed = ", ".join(d.value for d in AllocationDimension)
|
||||
raise Problem(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"Validation error",
|
||||
f"Неизвестное измерение: {value}. Допустимые: {allowed}",
|
||||
) from None
|
||||
|
||||
|
||||
async def _portfolio(session: SessionDep, portfolio_id: int) -> Portfolio:
|
||||
portfolio = await session.get(Portfolio, portfolio_id)
|
||||
if portfolio is None:
|
||||
raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет портфеля с id {portfolio_id}")
|
||||
return portfolio
|
||||
|
||||
|
||||
async def _targets_out(
|
||||
session: SessionDep, portfolio_id: int, dimension: AllocationDimension
|
||||
) -> TargetsOut:
|
||||
rows = (
|
||||
(
|
||||
await session.execute(
|
||||
select(PortfolioTarget)
|
||||
.where(
|
||||
PortfolioTarget.portfolio_id == portfolio_id,
|
||||
PortfolioTarget.dimension == dimension,
|
||||
)
|
||||
.order_by(PortfolioTarget.bucket)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
return TargetsOut(
|
||||
portfolio_id=portfolio_id,
|
||||
dimension=dimension.value,
|
||||
targets=[
|
||||
TargetOut(
|
||||
bucket=r.bucket,
|
||||
target_weight=Decimal(r.target_weight),
|
||||
band=None if r.band is None else Decimal(r.band),
|
||||
note=r.note,
|
||||
)
|
||||
for r in rows
|
||||
],
|
||||
weights_sum=sum((Decimal(r.target_weight) for r in rows), start=ZERO),
|
||||
)
|
||||
|
||||
|
||||
@router.get("/{portfolio_id}/targets", name="targets")
|
||||
async def get_targets(
|
||||
portfolio_id: int,
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
dimension: DimensionParam = DEFAULT_DIMENSION.value,
|
||||
) -> TargetsOut:
|
||||
await _portfolio(session, portfolio_id)
|
||||
return await _targets_out(session, portfolio_id, parse_dimension(dimension))
|
||||
|
||||
|
||||
@router.put("/{portfolio_id}/targets", name="set_targets")
|
||||
async def put_targets(
|
||||
portfolio_id: int, body: TargetsIn, session: SessionDep, _: CurrentUser
|
||||
) -> TargetsOut:
|
||||
"""Replace the whole set for one dimension.
|
||||
|
||||
The weights must add up to 1 within `WEIGHT_TOLERANCE`; otherwise the request is
|
||||
refused with the actual sum in the message, so the user can see by how much the plan
|
||||
misses rather than being handed a silently rescaled one.
|
||||
"""
|
||||
await _portfolio(session, portfolio_id)
|
||||
dimension = parse_dimension(body.dimension)
|
||||
|
||||
buckets = [t.bucket for t in body.targets]
|
||||
duplicates = sorted({b for b in buckets if buckets.count(b) > 1})
|
||||
if duplicates:
|
||||
raise Problem(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"Validation error",
|
||||
f"Бакет указан дважды: {', '.join(duplicates)}",
|
||||
)
|
||||
for target in body.targets:
|
||||
if target.target_weight < ZERO or target.target_weight > ONE:
|
||||
raise Problem(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"Validation error",
|
||||
f"Вес бакета «{target.bucket}» вне диапазона 0..1: {target.target_weight}",
|
||||
)
|
||||
if target.band is not None and target.band < ZERO:
|
||||
raise Problem(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"Validation error",
|
||||
f"Полоса допуска бакета «{target.bucket}» отрицательна: {target.band}",
|
||||
)
|
||||
|
||||
total = sum((t.target_weight for t in body.targets), start=ZERO)
|
||||
if body.targets and abs(total - ONE) > WEIGHT_TOLERANCE:
|
||||
raise Problem(
|
||||
status.HTTP_422_UNPROCESSABLE_ENTITY,
|
||||
"Validation error",
|
||||
f"Сумма весов должна быть 1, а не {format(total, 'f')}. "
|
||||
f"Сервер не нормализует веса — исправьте набор.",
|
||||
extra={"weights_sum": format(total, "f")},
|
||||
)
|
||||
|
||||
await session.execute(
|
||||
delete(PortfolioTarget).where(
|
||||
PortfolioTarget.portfolio_id == portfolio_id,
|
||||
PortfolioTarget.dimension == dimension,
|
||||
)
|
||||
)
|
||||
for target in body.targets:
|
||||
session.add(
|
||||
PortfolioTarget(
|
||||
portfolio_id=portfolio_id,
|
||||
dimension=dimension,
|
||||
bucket=target.bucket,
|
||||
target_weight=target.target_weight,
|
||||
band=target.band,
|
||||
note=target.note,
|
||||
)
|
||||
)
|
||||
await session.commit()
|
||||
return await _targets_out(session, portfolio_id, dimension)
|
||||
|
||||
|
||||
@router.get("/{portfolio_id}/rebalance", name="rebalance")
|
||||
async def get_rebalance(
|
||||
portfolio_id: int,
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
dimension: DimensionParam = DEFAULT_DIMENSION.value,
|
||||
cash_available: Annotated[
|
||||
Decimal | None, Query(description="переопределяет остаток на счетах для what-if")
|
||||
] = None,
|
||||
) -> RebalanceOut:
|
||||
await _portfolio(session, portfolio_id)
|
||||
plan = await compute_rebalance(
|
||||
session,
|
||||
portfolio_id,
|
||||
parse_dimension(dimension),
|
||||
cash_available_rub=cash_available,
|
||||
)
|
||||
return RebalanceOut(
|
||||
portfolio_id=plan.portfolio_id,
|
||||
dimension=plan.dimension.value,
|
||||
as_of=plan.as_of,
|
||||
total_value_rub=plan.total_value_rub,
|
||||
cash_available_rub=plan.cash_available_rub,
|
||||
buckets=[
|
||||
RebalanceBucketOut(
|
||||
bucket=b.bucket,
|
||||
current_value_rub=b.current_value_rub,
|
||||
current_weight=b.current_weight,
|
||||
target_weight=b.target_weight,
|
||||
drift=b.drift,
|
||||
within_band=b.within_band,
|
||||
delta_value_rub=b.delta_value_rub,
|
||||
trades=[
|
||||
TradeOut(
|
||||
instrument_id=t.instrument_id,
|
||||
ticker=t.ticker,
|
||||
name=t.name,
|
||||
action=t.action,
|
||||
# inside the band nothing is proposed, so the quantity is NULL
|
||||
suggested_qty=None if b.within_band else t.qty,
|
||||
lot=t.lot,
|
||||
price=t.price,
|
||||
price_currency=t.price_currency,
|
||||
amount_rub=t.amount_rub,
|
||||
blocked_by_cash=t.blocked_by_cash,
|
||||
)
|
||||
for t in b.trades
|
||||
],
|
||||
)
|
||||
for b in plan.buckets
|
||||
],
|
||||
warnings=plan.warnings,
|
||||
)
|
||||
|
||||
|
||||
__all__ = ["router"]
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Tax: the year's summary, and the open lots with their ЛДВ dates.
|
||||
|
||||
`GET /tax` reads `metric_tax_year`, which the refresh built. `GET /tax/lots` is computed per
|
||||
request on purpose — it depends on today's date and today's price, and a nightly table would
|
||||
answer `days_to_ldv` as of last night, which is off by one on precisely the day the number
|
||||
is being consulted.
|
||||
|
||||
Both responses carry `estimated: true`, the rate that produced them and a disclaimer. The
|
||||
broker is the tax agent; see `analytics/tax.py` for what is and is not modelled.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict
|
||||
from decimal import Decimal
|
||||
from typing import Annotated
|
||||
|
||||
from fastapi import APIRouter, Query
|
||||
from sqlalchemy import select
|
||||
|
||||
from fintracker.analytics import today_local
|
||||
from fintracker.analytics.tax import DISCLAIMER, TAX_RATE, money, open_lot_tax
|
||||
from fintracker.api.deps import CurrentUser, SessionDep
|
||||
from fintracker.api.schemas.tax import (
|
||||
TaxAccountOut,
|
||||
TaxLotOut,
|
||||
TaxLotsOut,
|
||||
TaxTotals,
|
||||
TaxYearOut,
|
||||
)
|
||||
from fintracker.models import Account, MetricTaxYear
|
||||
|
||||
router = APIRouter(prefix="/tax", tags=["tax"])
|
||||
|
||||
ZERO = Decimal(0)
|
||||
|
||||
TOTAL_FIELDS = (
|
||||
"dividends_gross_rub",
|
||||
"coupons_gross_rub",
|
||||
"tax_withheld_rub",
|
||||
"realized_gain_rub",
|
||||
"realized_loss_rub",
|
||||
"ldv_exempt_rub",
|
||||
"taxable_base_rub",
|
||||
"estimated_tax_rub",
|
||||
)
|
||||
|
||||
|
||||
@router.get("", name="summary")
|
||||
async def tax_year(
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
year: Annotated[int | None, Query(ge=1990, le=2200)] = None,
|
||||
account_id: int | None = None,
|
||||
) -> TaxYearOut:
|
||||
"""The year's estimated tax position, per account and in total."""
|
||||
year = year or today_local().year
|
||||
conditions = [MetricTaxYear.year == year]
|
||||
if account_id is not None:
|
||||
conditions.append(MetricTaxYear.account_id == account_id)
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(MetricTaxYear, Account.name)
|
||||
.join(Account, Account.id == MetricTaxYear.account_id)
|
||||
.where(*conditions)
|
||||
.order_by(Account.name)
|
||||
)
|
||||
).all()
|
||||
|
||||
accounts = [
|
||||
TaxAccountOut(
|
||||
account_id=r[0].account_id,
|
||||
account_name=r[1],
|
||||
**{f: getattr(r[0], f) for f in TOTAL_FIELDS},
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
# the rate is read back from the rows rather than assumed, so a year computed under an
|
||||
# older rate keeps reporting the rate it was computed with
|
||||
rate = rows[0][0].tax_rate if rows else TAX_RATE
|
||||
return TaxYearOut(
|
||||
year=year,
|
||||
tax_rate=rate,
|
||||
accounts=accounts,
|
||||
totals=TaxTotals(
|
||||
**{f: money(sum((getattr(a, f) for a in accounts), start=ZERO)) for f in TOTAL_FIELDS}
|
||||
),
|
||||
disclaimer=DISCLAIMER,
|
||||
)
|
||||
|
||||
|
||||
@router.get("/lots", name="lots")
|
||||
async def tax_lots(
|
||||
session: SessionDep,
|
||||
_: CurrentUser,
|
||||
year: Annotated[int | None, Query(ge=1990, le=2200)] = None,
|
||||
account_id: int | None = None,
|
||||
) -> TaxLotsOut:
|
||||
"""Open lots with the date after which a sale falls under the long-term exemption.
|
||||
|
||||
`year` selects the rate the "what if I sold today" figure is computed at; the lots
|
||||
themselves are always the ones open right now. `lot` is rebuilt from the ledger on every
|
||||
refresh and holds today's state only, so there is no honest way to answer "which lots
|
||||
were open on 31 December two years ago" — and inventing one would be worse than the
|
||||
limitation.
|
||||
"""
|
||||
as_of = today_local()
|
||||
year = year or as_of.year
|
||||
lots = await open_lot_tax(session, as_of=as_of, account_id=account_id)
|
||||
return TaxLotsOut(
|
||||
year=year,
|
||||
as_of=as_of,
|
||||
tax_rate=TAX_RATE,
|
||||
lots=[TaxLotOut(**asdict(lot)) for lot in lots],
|
||||
disclaimer=DISCLAIMER,
|
||||
)
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Schemas for the benchmark list and the portfolio-vs-index comparison (phase-4 contract §3).
|
||||
|
||||
`kind` crosses the wire as a plain string (`price` | `total_return`), like every other stable
|
||||
key in this API — and unlike a Dart enum, which the generated client cannot always name. The
|
||||
client is expected to *use* it: a portfolio compared against a price index is being compared
|
||||
against a series that throws its dividends away, and the screen has to say so.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
from fintracker.api.schemas.common import MoneyOpt
|
||||
|
||||
|
||||
class BenchmarkOut(BaseModel):
|
||||
id: int
|
||||
code: str
|
||||
"""MOEX secid or another stable code: IMOEX, MCFTR, RGBITR."""
|
||||
name: str
|
||||
kind: str
|
||||
"""price | total_return — a price index understates a holder's result by its dividends."""
|
||||
source: str
|
||||
"""moex | manual"""
|
||||
currency: str
|
||||
is_default: bool
|
||||
is_active: bool
|
||||
instrument_id: int | None
|
||||
"""The instrument carrying the index history; null until the index has been synced."""
|
||||
history_from: date | None
|
||||
history_to: date | None
|
||||
"""Range actually present in `price_daily`; null when there is no history at all."""
|
||||
|
||||
|
||||
class BenchmarkCreate(BaseModel):
|
||||
code: str = Field(min_length=1, max_length=32)
|
||||
name: str = Field(min_length=1, max_length=128)
|
||||
kind: str
|
||||
"""price | total_return"""
|
||||
source: str = "moex"
|
||||
currency: str = "RUB"
|
||||
is_default: bool = False
|
||||
is_active: bool = True
|
||||
instrument_id: int | None = None
|
||||
|
||||
|
||||
class BenchmarkPatch(BaseModel):
|
||||
code: str | None = Field(default=None, min_length=1, max_length=32)
|
||||
name: str | None = Field(default=None, min_length=1, max_length=128)
|
||||
kind: str | None = None
|
||||
source: str | None = None
|
||||
currency: str | None = None
|
||||
is_default: bool | None = None
|
||||
is_active: bool | None = None
|
||||
instrument_id: int | None = None
|
||||
|
||||
|
||||
class BenchmarkReturnOut(BaseModel):
|
||||
benchmark_id: int
|
||||
code: str
|
||||
name: str
|
||||
kind: str
|
||||
twr: MoneyOpt
|
||||
"""Cumulative over the period; null when the index had no quote to start from."""
|
||||
twr_annualized: MoneyOpt
|
||||
days_skipped: int
|
||||
"""Days of the compared window the index had no quote for. Non-zero means the two series
|
||||
are not day-for-day comparable, and the client must show it."""
|
||||
excess: MoneyOpt
|
||||
"""portfolio_twr - twr; null when either side is unknown."""
|
||||
|
||||
|
||||
class BenchmarkComparisonOut(BaseModel):
|
||||
period: str
|
||||
"""1m | 3m | 6m | ytd | 1y | 3y | all"""
|
||||
date_from: date
|
||||
date_to: date
|
||||
portfolio_twr: MoneyOpt
|
||||
portfolio_twr_annualized: MoneyOpt
|
||||
portfolio_days_skipped: int
|
||||
benchmarks: list[BenchmarkReturnOut]
|
||||
|
||||
|
||||
class BenchmarkComparison(BaseModel):
|
||||
rows: list[BenchmarkComparisonOut]
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Wire shapes for goals and their progress (docs/ai/phase4-contract.md §4)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from fintracker.api.schemas.common import Money, MoneyOpt
|
||||
|
||||
|
||||
class GoalOut(BaseModel):
|
||||
id: int
|
||||
name: str
|
||||
scope: str
|
||||
target_amount: Money
|
||||
currency: str
|
||||
target_date: date | None
|
||||
monthly_contribution: MoneyOpt
|
||||
note: str | None
|
||||
archived: bool
|
||||
|
||||
|
||||
class GoalCreate(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str = Field(min_length=1, max_length=128)
|
||||
scope: str = Field(default="all", max_length=32)
|
||||
target_amount: Money
|
||||
currency: str = Field(default="RUB", min_length=3, max_length=3)
|
||||
target_date: date | None = None
|
||||
monthly_contribution: MoneyOpt = None
|
||||
note: str | None = None
|
||||
archived: bool = False
|
||||
|
||||
|
||||
class GoalPatch(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
name: str | None = Field(default=None, min_length=1, max_length=128)
|
||||
scope: str | None = Field(default=None, max_length=32)
|
||||
target_amount: MoneyOpt = None
|
||||
currency: str | None = Field(default=None, min_length=3, max_length=3)
|
||||
target_date: date | None = None
|
||||
monthly_contribution: MoneyOpt = None
|
||||
note: str | None = None
|
||||
archived: bool | None = None
|
||||
|
||||
|
||||
class GoalProgressOut(BaseModel):
|
||||
goal_id: int
|
||||
as_of: date
|
||||
current_value_rub: Money
|
||||
target_amount_rub: Money
|
||||
progress: Money
|
||||
projected_date: date | None
|
||||
"""NULL means the current trend does not reach the target — never a far-future date."""
|
||||
basis: str
|
||||
"""xirr | contribution | none"""
|
||||
assumed_rate: MoneyOpt
|
||||
monthly_needed_rub: MoneyOpt
|
||||
"""NULL when the goal has no deadline, or the deadline has already passed."""
|
||||
on_track: bool | None
|
||||
@@ -0,0 +1,84 @@
|
||||
"""Schemas for `/income` — the dividend and coupon calendar (phase-4 contract §1).
|
||||
|
||||
`basis` is a plain string here and on every row, never an enum and never optional: a calendar
|
||||
that shows «12 400 ₽ ожидается» without saying whether that is a published coupon schedule or
|
||||
an extrapolation from two payments is not a forecast the user can audit. The client is
|
||||
required to render it.
|
||||
|
||||
`amount_rub` is nullable for the usual reason — the payment's date had no CBR rate — and the
|
||||
client shows that as «нет курса», not as 0 ₽.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fintracker.api.schemas.common import Money, MoneyOpt
|
||||
|
||||
|
||||
class CalendarEntry(BaseModel):
|
||||
instrument_id: int
|
||||
ticker: str | None
|
||||
name: str
|
||||
kind: str
|
||||
"""dividend | coupon | amortization | repayment"""
|
||||
expected_date: date
|
||||
record_date: date | None
|
||||
qty: Money
|
||||
"""Position the amount was computed on; 0 on a historical row, which carries its own sum."""
|
||||
per_unit: MoneyOpt
|
||||
amount: Money
|
||||
currency: str
|
||||
amount_rub: MoneyOpt
|
||||
"""Null when the date has no rate — never a substituted number."""
|
||||
basis: str
|
||||
"""schedule | announced | history | paid — where the number came from."""
|
||||
tax_withheld: MoneyOpt
|
||||
|
||||
|
||||
class CalendarOut(BaseModel):
|
||||
as_of: date
|
||||
currency: str
|
||||
total_expected_rub: Money
|
||||
"""Sum of the FUTURE entries only; paid rows never inflate an expectation."""
|
||||
entries: list[CalendarEntry]
|
||||
by_basis: dict[str, Money]
|
||||
"""Expected RUB split by basis. Sums that mix `schedule` with `history` hide the guess."""
|
||||
|
||||
|
||||
class HistoryRow(BaseModel):
|
||||
month: date
|
||||
"""First day of the month."""
|
||||
kind: str
|
||||
currency: str
|
||||
amount: Money
|
||||
amount_rub: MoneyOpt
|
||||
tax_withheld: Money
|
||||
payment_count: int
|
||||
|
||||
|
||||
class HistoryTotals(BaseModel):
|
||||
amount_rub: Money
|
||||
tax_withheld_rub: Money
|
||||
|
||||
|
||||
class HistoryOut(BaseModel):
|
||||
rows: list[HistoryRow]
|
||||
totals: HistoryTotals
|
||||
|
||||
|
||||
class ForecastMonth(BaseModel):
|
||||
month: date
|
||||
amount_rub: Money
|
||||
by_basis: dict[str, Money]
|
||||
|
||||
|
||||
class ForecastOut(BaseModel):
|
||||
months: list[ForecastMonth]
|
||||
total_rub: Money
|
||||
annual_yield_on_value: MoneyOpt
|
||||
"""Expected 12-month income over the scope's current value; null without a valuation."""
|
||||
warnings: list[str]
|
||||
"""What the forecast could not cover, verbatim from the data-quality findings."""
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Wire shapes for target weights and rebalancing (docs/ai/phase4-contract.md §2).
|
||||
|
||||
`dimension`, `bucket` and `action` are plain strings, never enums: `AssetClass` must not
|
||||
leak into the generated Dart client (AGENTS.md), and the same rule keeps every stable key
|
||||
a string. Weights, drifts and quantities are Decimals serialised as strings — a weight is
|
||||
`"0.6"`, not `60`.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
|
||||
from fintracker.api.schemas.common import Money, MoneyOpt
|
||||
|
||||
|
||||
class TargetIn(BaseModel):
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
bucket: str = Field(min_length=1, max_length=64)
|
||||
target_weight: Money
|
||||
band: MoneyOpt = None
|
||||
"""Tolerance in the same units as the weight: `"0.05"` is ±5 pp, not 5 %."""
|
||||
note: str | None = None
|
||||
|
||||
|
||||
class TargetsIn(BaseModel):
|
||||
"""A complete set for one dimension. Partial updates are not supported: weights only
|
||||
mean anything together, and a half-written set could not be checked against 1."""
|
||||
|
||||
model_config = ConfigDict(extra="forbid")
|
||||
|
||||
dimension: str = Field(description="asset_class | sector | country | currency")
|
||||
targets: list[TargetIn]
|
||||
|
||||
|
||||
class TargetOut(BaseModel):
|
||||
bucket: str
|
||||
target_weight: Money
|
||||
band: MoneyOpt
|
||||
note: str | None
|
||||
|
||||
|
||||
class TargetsOut(BaseModel):
|
||||
portfolio_id: int
|
||||
dimension: str
|
||||
targets: list[TargetOut]
|
||||
weights_sum: Money
|
||||
"""What the weights actually add up to. The server never normalises them."""
|
||||
|
||||
|
||||
class TradeOut(BaseModel):
|
||||
instrument_id: int
|
||||
ticker: str
|
||||
name: str
|
||||
action: str
|
||||
"""buy | sell"""
|
||||
suggested_qty: MoneyOpt
|
||||
"""Whole lots; NULL when the instrument has no usable price."""
|
||||
lot: int | None
|
||||
price: MoneyOpt
|
||||
price_currency: str | None
|
||||
amount_rub: Money
|
||||
blocked_by_cash: bool
|
||||
|
||||
|
||||
class RebalanceBucketOut(BaseModel):
|
||||
bucket: str
|
||||
current_value_rub: Money
|
||||
current_weight: Money
|
||||
target_weight: MoneyOpt
|
||||
drift: MoneyOpt
|
||||
"""current - target, in fractions of the whole portfolio."""
|
||||
within_band: bool
|
||||
delta_value_rub: Money
|
||||
trades: list[TradeOut]
|
||||
|
||||
|
||||
class RebalanceOut(BaseModel):
|
||||
portfolio_id: int
|
||||
dimension: str
|
||||
as_of: date
|
||||
total_value_rub: Money
|
||||
cash_available_rub: Money
|
||||
buckets: list[RebalanceBucketOut]
|
||||
warnings: list[str]
|
||||
@@ -0,0 +1,86 @@
|
||||
"""Schemas for the tax screens (phase-4 contract §5).
|
||||
|
||||
Every response carries `estimated: true`, the `tax_rate` that produced it and a `disclaimer`,
|
||||
and none of the three is optional. The broker is the tax agent; these numbers exist so its
|
||||
certificate can be checked and so the cost of selling before the three-year mark is visible
|
||||
*before* the sale, not to replace it. A client that drops the marking is showing a number it
|
||||
is not entitled to show.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
from fintracker.api.schemas.common import Money, MoneyOpt
|
||||
|
||||
|
||||
class TaxTotals(BaseModel):
|
||||
dividends_gross_rub: Money
|
||||
coupons_gross_rub: Money
|
||||
tax_withheld_rub: Money
|
||||
"""What the broker already held back on those payments."""
|
||||
realized_gain_rub: Money
|
||||
"""Sum of the profitable disposals of the year, LDV ones included."""
|
||||
realized_loss_rub: Money
|
||||
"""Sum of the losing ones, negative."""
|
||||
ldv_exempt_rub: Money
|
||||
"""Net result of the disposals exempt under art. 219.1, taken back out of the base.
|
||||
Negative if those lots lost money: the exemption removes the whole result, not the
|
||||
profitable half."""
|
||||
taxable_base_rub: Money
|
||||
"""max(0, gain + loss - ldv_exempt). Dividends and coupons are not in it — the agent
|
||||
withholds on them at source."""
|
||||
estimated_tax_rub: Money
|
||||
|
||||
|
||||
class TaxAccountOut(TaxTotals):
|
||||
account_id: int
|
||||
account_name: str
|
||||
|
||||
|
||||
class TaxYearOut(BaseModel):
|
||||
year: int
|
||||
estimated: bool = True
|
||||
"""Always true. There is no mode in which this endpoint returns an authoritative figure."""
|
||||
tax_rate: Money
|
||||
"""The rate actually applied, as a fraction: "0.13", never 13."""
|
||||
accounts: list[TaxAccountOut]
|
||||
totals: TaxTotals
|
||||
disclaimer: str
|
||||
|
||||
|
||||
class TaxLotOut(BaseModel):
|
||||
lot_id: int
|
||||
instrument_id: int
|
||||
ticker: str | None
|
||||
name: str
|
||||
account_id: int
|
||||
account_name: str
|
||||
open_date: date
|
||||
qty_remaining: Money
|
||||
cost_rub: MoneyOpt
|
||||
"""Cost at the CBR rate of the lot's own open date, prorated to the part still held;
|
||||
null when that day had no rate."""
|
||||
market_value_rub: MoneyOpt
|
||||
"""Null when the instrument has no price — never zero."""
|
||||
unrealized_gain_rub: MoneyOpt
|
||||
ldv_eligible: bool
|
||||
"""Held three full years on an exchange-traded instrument (art. 219.1) — an estimate:
|
||||
the classic rule only, MOEX papers only, ИИС regimes not modelled."""
|
||||
ldv_date: date | None
|
||||
"""First day a sale qualifies; null for an instrument that cannot qualify at all."""
|
||||
days_to_ldv: int | None
|
||||
"""Calendar days left, 0 once eligible; null alongside a null `ldv_date`."""
|
||||
tax_if_sold_now_rub: MoneyOpt
|
||||
"""0 for an eligible lot; null when the position cannot be valued."""
|
||||
|
||||
|
||||
class TaxLotsOut(BaseModel):
|
||||
year: int
|
||||
as_of: date
|
||||
estimated: bool = True
|
||||
tax_rate: Money
|
||||
lots: list[TaxLotOut]
|
||||
disclaimer: str
|
||||
@@ -25,19 +25,33 @@ from fintracker.models.ledger import (
|
||||
)
|
||||
from fintracker.models.metrics import (
|
||||
AllocationDimension,
|
||||
IncomeBasis,
|
||||
MetricAllocation,
|
||||
MetricBenchmarkReturns,
|
||||
MetricCashFlowBroker,
|
||||
MetricCashFlowMonthly,
|
||||
MetricDataQuality,
|
||||
MetricGoalProgress,
|
||||
MetricHolding,
|
||||
MetricIncomeCalendar,
|
||||
MetricIncomeMonthly,
|
||||
MetricNetWorthDaily,
|
||||
MetricPortfolioValueDaily,
|
||||
MetricRebalance,
|
||||
MetricRefreshLog,
|
||||
MetricReturns,
|
||||
MetricRunway,
|
||||
MetricSpendingByCategory,
|
||||
MetricTaxYear,
|
||||
)
|
||||
from fintracker.models.planning import (
|
||||
Benchmark,
|
||||
BenchmarkKind,
|
||||
Goal,
|
||||
PortfolioTarget,
|
||||
)
|
||||
from fintracker.models.pricing import (
|
||||
BondNominalSchedule,
|
||||
CashSnapshot,
|
||||
CorporateAction,
|
||||
CorporateActionKind,
|
||||
@@ -95,6 +109,9 @@ __all__ = [
|
||||
"AllocationDimension",
|
||||
"AppUser",
|
||||
"AssetClass",
|
||||
"Benchmark",
|
||||
"BenchmarkKind",
|
||||
"BondNominalSchedule",
|
||||
"Broker",
|
||||
"CashSnapshot",
|
||||
"CashTxn",
|
||||
@@ -111,6 +128,8 @@ __all__ = [
|
||||
"FlowLinkKind",
|
||||
"FlowType",
|
||||
"FxRateDaily",
|
||||
"Goal",
|
||||
"IncomeBasis",
|
||||
"Instrument",
|
||||
"InstrumentAlias",
|
||||
"JobStatus",
|
||||
@@ -118,20 +137,27 @@ __all__ = [
|
||||
"LotDisposal",
|
||||
"Merchant",
|
||||
"MetricAllocation",
|
||||
"MetricBenchmarkReturns",
|
||||
"MetricCashFlowBroker",
|
||||
"MetricCashFlowMonthly",
|
||||
"MetricDataQuality",
|
||||
"MetricGoalProgress",
|
||||
"MetricHolding",
|
||||
"MetricIncomeCalendar",
|
||||
"MetricIncomeMonthly",
|
||||
"MetricNetWorthDaily",
|
||||
"MetricPortfolioValueDaily",
|
||||
"MetricRebalance",
|
||||
"MetricRefreshLog",
|
||||
"MetricReturns",
|
||||
"MetricRunway",
|
||||
"MetricSpendingByCategory",
|
||||
"MetricTaxYear",
|
||||
"PendingInstrument",
|
||||
"PendingInstrumentStatus",
|
||||
"Portfolio",
|
||||
"PortfolioAccount",
|
||||
"PortfolioTarget",
|
||||
"PositionSnapshot",
|
||||
"PriceCoverage",
|
||||
"PriceDaily",
|
||||
|
||||
@@ -8,7 +8,7 @@ from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy import Boolean, ForeignKey, Integer, String, Text, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, db_enum
|
||||
@@ -284,4 +284,195 @@ class MetricAllocation(Base):
|
||||
"""Share of the scope's valued total; the weights of one dimension add up to 1."""
|
||||
holding_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
"""Instruments in this bucket; 0 for the cash bucket."""
|
||||
target_weight: Mapped[Decimal | None]
|
||||
"""From `portfolio_target`; NULL when the user set no target for this bucket."""
|
||||
drift: Mapped[Decimal | None]
|
||||
"""`weight - target_weight` in percentage points of the whole, NULL without a target."""
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class IncomeBasis(enum.StrEnum):
|
||||
"""Where a calendar entry's number comes from — the honesty column.
|
||||
|
||||
A bond's future coupon is arithmetic on a published schedule; an announced dividend is a
|
||||
fact the issuer stated; a projected dividend is an extrapolation from history and can be
|
||||
wrong by any amount. Mixing the three into one «expected income» figure is the fastest
|
||||
way to make a forecast that nobody can audit.
|
||||
"""
|
||||
|
||||
schedule = "schedule"
|
||||
"""Bond coupon from the payment schedule and the nominal in force on that date."""
|
||||
announced = "announced"
|
||||
"""Declared by the issuer: a `corporate_action` with a future record date."""
|
||||
history = "history"
|
||||
"""Extrapolated from the last 24 months — a guess, labelled as one."""
|
||||
paid = "paid"
|
||||
"""Already received; the entry is history, not a forecast."""
|
||||
|
||||
|
||||
class MetricIncomeCalendar(Base):
|
||||
"""Dividends and coupons, past and expected, per scope and instrument (plan §3).
|
||||
|
||||
Rows are per payment, not per month, because the screen the user actually wants answers
|
||||
«what lands, and when» — and because an amount that turns out wrong has to be traceable
|
||||
to the single assumption that produced it (`basis`, `per_unit`, `qty`).
|
||||
"""
|
||||
|
||||
__tablename__ = "metric_income_calendar"
|
||||
__table_args__ = (UniqueConstraint("scope", "instrument_id", "kind", "expected_date", "basis"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
scope: Mapped[str] = mapped_column(String(32), index=True)
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
kind: Mapped[str] = mapped_column(String(16))
|
||||
"""dividend | coupon | amortization | repayment — the `corporate_action` kinds that pay."""
|
||||
expected_date: Mapped[date] = mapped_column(index=True)
|
||||
record_date: Mapped[date | None]
|
||||
qty: Mapped[Decimal]
|
||||
"""Position held when the payment is expected; 0 once the position is gone."""
|
||||
per_unit: Mapped[Decimal | None]
|
||||
amount: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
amount_rub: Mapped[Decimal | None]
|
||||
"""NULL when the date has no FX rate — never a substituted number."""
|
||||
basis: Mapped[IncomeBasis] = mapped_column(db_enum(IncomeBasis, "income_basis"))
|
||||
tax_withheld: Mapped[Decimal | None]
|
||||
"""What the broker already held back, for payments that happened."""
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class MetricIncomeMonthly(Base):
|
||||
"""Income actually received, per scope, month and kind — the history half of the screen."""
|
||||
|
||||
__tablename__ = "metric_income_monthly"
|
||||
__table_args__ = (UniqueConstraint("scope", "month", "kind", "currency"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
scope: Mapped[str] = mapped_column(String(32), index=True)
|
||||
month: Mapped[date]
|
||||
"""First day of the month."""
|
||||
kind: Mapped[str] = mapped_column(String(16))
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
amount: Mapped[Decimal]
|
||||
amount_rub: Mapped[Decimal | None]
|
||||
tax_withheld: Mapped[Decimal]
|
||||
payment_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class MetricBenchmarkReturns(Base):
|
||||
"""A benchmark's TWR over the same period and date grid as `metric_returns`.
|
||||
|
||||
Kept beside the portfolio's own returns rather than inside them: the set of benchmarks is
|
||||
a user's choice, and a column per index would mean a migration per index. Comparing
|
||||
requires the same grid — a benchmark measured over a different set of days is not a
|
||||
comparison, so `date_from`/`date_to` are stored and checked.
|
||||
"""
|
||||
|
||||
__tablename__ = "metric_benchmark_returns"
|
||||
__table_args__ = (UniqueConstraint("scope", "period", "benchmark_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
scope: Mapped[str] = mapped_column(String(32), index=True)
|
||||
period: Mapped[str] = mapped_column(String(8))
|
||||
benchmark_id: Mapped[int] = mapped_column(ForeignKey("benchmark.id", ondelete="CASCADE"))
|
||||
date_from: Mapped[date]
|
||||
date_to: Mapped[date]
|
||||
twr: Mapped[Decimal | None]
|
||||
twr_annualized: Mapped[Decimal | None]
|
||||
days_skipped: Mapped[int] = mapped_column(Integer, default=0)
|
||||
"""Days the index had no quote for. Non-zero means the comparison is not like-for-like."""
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class MetricRebalance(Base):
|
||||
"""What to buy or sell to reach the target weights (plan §3).
|
||||
|
||||
`suggested_qty` respects the lot size and the cash on hand, because a recommendation that
|
||||
cannot be executed is not a recommendation. It is whole lots, never a fractional share.
|
||||
"""
|
||||
|
||||
__tablename__ = "metric_rebalance"
|
||||
__table_args__ = (UniqueConstraint("portfolio_id", "dimension", "bucket", "instrument_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
portfolio_id: Mapped[int] = mapped_column(ForeignKey("portfolio.id", ondelete="CASCADE"))
|
||||
dimension: Mapped[AllocationDimension] = mapped_column(
|
||||
db_enum(AllocationDimension, "allocation_dimension")
|
||||
)
|
||||
bucket: Mapped[str] = mapped_column(String(64))
|
||||
instrument_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE")
|
||||
)
|
||||
"""NULL on the bucket's own summary row; set on each proposed trade inside it."""
|
||||
current_value_rub: Mapped[Decimal]
|
||||
current_weight: Mapped[Decimal]
|
||||
target_weight: Mapped[Decimal | None]
|
||||
delta_value_rub: Mapped[Decimal]
|
||||
"""Positive: buy this much. Negative: sell."""
|
||||
suggested_qty: Mapped[Decimal | None]
|
||||
"""Whole lots; NULL when the instrument has no usable price."""
|
||||
lot: Mapped[int | None] = mapped_column(Integer)
|
||||
price: Mapped[Decimal | None]
|
||||
price_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
within_band: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
"""True when the drift is inside `portfolio_target.band` — shown, but not acted on."""
|
||||
blocked_by_cash: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
"""A buy the available cash does not cover; the quantity is what the cash does allow."""
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class MetricTaxYear(Base):
|
||||
"""The tax picture per calendar year and account (plan §3, ст. 214.1 / 219.1 НК).
|
||||
|
||||
Estimated, and labelled estimated everywhere it surfaces: the broker is the tax agent and
|
||||
the authoritative number is its own certificate. This exists to make that certificate
|
||||
checkable and to show the cost of selling a position before the three-year mark.
|
||||
"""
|
||||
|
||||
__tablename__ = "metric_tax_year"
|
||||
__table_args__ = (UniqueConstraint("year", "account_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
year: Mapped[int] = mapped_column(Integer, index=True)
|
||||
account_id: Mapped[int] = mapped_column(ForeignKey("account.id", ondelete="CASCADE"))
|
||||
dividends_gross_rub: Mapped[Decimal]
|
||||
coupons_gross_rub: Mapped[Decimal]
|
||||
tax_withheld_rub: Mapped[Decimal]
|
||||
"""What the broker already took — the difference between gross and what arrived."""
|
||||
realized_gain_rub: Mapped[Decimal]
|
||||
"""FIFO result in roubles, each leg at the CBR rate of its own date (currency revaluation
|
||||
is part of the base, plan §7 вопрос 4)."""
|
||||
realized_loss_rub: Mapped[Decimal]
|
||||
ldv_exempt_rub: Mapped[Decimal]
|
||||
"""Gain on lots held 3+ years, exempt under art. 219.1 — an estimate, see the docstring."""
|
||||
taxable_base_rub: Mapped[Decimal]
|
||||
estimated_tax_rub: Mapped[Decimal]
|
||||
tax_rate: Mapped[Decimal]
|
||||
"""The rate applied, so a changed rate is visible rather than baked into the number."""
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class MetricGoalProgress(Base):
|
||||
"""Where a goal stands and when it is projected to be met."""
|
||||
|
||||
__tablename__ = "metric_goal_progress"
|
||||
|
||||
goal_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("goal.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
as_of: Mapped[date]
|
||||
current_value_rub: Mapped[Decimal]
|
||||
target_amount_rub: Mapped[Decimal]
|
||||
progress: Mapped[Decimal]
|
||||
"""current / target, clamped to [0, …]; can exceed 1 for a goal already met."""
|
||||
projected_date: Mapped[date | None]
|
||||
"""NULL when the trend does not reach the target (flat or falling), never a far-future
|
||||
date pretending to be an answer."""
|
||||
basis: Mapped[str] = mapped_column(String(16))
|
||||
"""xirr | contribution | none — which assumption produced `projected_date`."""
|
||||
assumed_rate: Mapped[Decimal | None]
|
||||
monthly_needed_rub: Mapped[Decimal | None]
|
||||
"""Contribution required to hit `target_date`; NULL when the goal has no deadline."""
|
||||
on_track: Mapped[bool | None]
|
||||
computed_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""What the portfolio is *supposed* to look like: targets, benchmarks, goals (plan §4, фаза 4).
|
||||
|
||||
Everything else in `models/` records what happened. These three record intent, and that is
|
||||
why they are separate: a target weight is not a measurement, it is a decision the user made,
|
||||
and a refresh that rebuilds every metric from scratch must never touch it.
|
||||
|
||||
* `PortfolioTarget` — the desired split along one allocation dimension. Rebalancing compares
|
||||
it against `metric_allocation` and proposes trades.
|
||||
* `Benchmark` — an index the portfolio is measured against. Stored as a row rather than a
|
||||
constant because the comparison list is a user's choice, and because an index quoted on
|
||||
MOEX is an ordinary `instrument` whose history already lands in `price_daily`.
|
||||
* `Goal` — an amount by a date. Progress is derived, never stored here.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, String, Text, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
from fintracker.models.metrics import AllocationDimension
|
||||
|
||||
|
||||
class PortfolioTarget(TimestampMixin, Base):
|
||||
"""Target weight of one bucket of one dimension, as a fraction (0.25 = 25 %).
|
||||
|
||||
Keyed by the same `(dimension, bucket)` pair `metric_allocation` uses, so the two join
|
||||
directly and a bucket that exists in only one of them is visible rather than silently
|
||||
dropped — an allocation that drifted to zero still owes an answer.
|
||||
"""
|
||||
|
||||
__tablename__ = "portfolio_target"
|
||||
__table_args__ = (UniqueConstraint("portfolio_id", "dimension", "bucket"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
portfolio_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("portfolio.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
dimension: Mapped[AllocationDimension] = mapped_column(
|
||||
db_enum(AllocationDimension, "allocation_dimension")
|
||||
)
|
||||
bucket: Mapped[str] = mapped_column(String(64))
|
||||
target_weight: Mapped[Decimal]
|
||||
"""Fraction of the portfolio. Weights of one dimension should add to 1; the API reports
|
||||
the shortfall rather than normalising, because a total of 0.9 is a mistake, not a scale."""
|
||||
band: Mapped[Decimal | None]
|
||||
"""Tolerance around the target below which no rebalancing is suggested (0.05 = ±5 pp).
|
||||
Without it every price tick produces a trade recommendation."""
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class BenchmarkKind(enum.StrEnum):
|
||||
price = "price"
|
||||
"""Price index — no dividends (IMOEX). Understates what a holder actually earned."""
|
||||
total_return = "total_return"
|
||||
"""Gross total-return index — dividends reinvested (MCFTR). The honest comparison."""
|
||||
|
||||
|
||||
class Benchmark(TimestampMixin, Base):
|
||||
"""An index the portfolio's TWR is compared against."""
|
||||
|
||||
__tablename__ = "benchmark"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
code: Mapped[str] = mapped_column(String(32), unique=True)
|
||||
"""MOEX secid or another stable code: IMOEX, MCFTR, RGBITR."""
|
||||
name: Mapped[str] = mapped_column(String(128))
|
||||
kind: Mapped[BenchmarkKind] = mapped_column(db_enum(BenchmarkKind, "benchmark_kind"))
|
||||
instrument_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="SET NULL")
|
||||
)
|
||||
"""The `instrument` carrying the index's price history; NULL until it is first synced."""
|
||||
source: Mapped[str] = mapped_column(String(16), default="moex")
|
||||
"""moex | manual — the S&P 500 has no free official feed (plan §7, вопрос 2)."""
|
||||
currency: Mapped[str] = mapped_column(String(3), default="RUB")
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
"""Shown next to the portfolio's own return without being asked for."""
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
|
||||
|
||||
class Goal(TimestampMixin, Base):
|
||||
"""An amount to reach, optionally by a date. Progress and the projected date are derived.
|
||||
|
||||
`scope` is the same string the analytics API uses (`all`, `account:<id>`,
|
||||
`portfolio:<id>`) so a goal can track one account or the whole net worth without a
|
||||
second way of naming a subset.
|
||||
"""
|
||||
|
||||
__tablename__ = "goal"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True)
|
||||
scope: Mapped[str] = mapped_column(String(32), default="all")
|
||||
target_amount: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3), default="RUB")
|
||||
target_date: Mapped[date | None]
|
||||
"""NULL means «no deadline» — progress is still tracked, only the date is projected."""
|
||||
monthly_contribution: Mapped[Decimal | None]
|
||||
"""Planned top-up. Used for the projection when the goal has too little history for a
|
||||
trailing XIRR to mean anything."""
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
archived: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
@@ -134,6 +134,29 @@ class CorporateAction(TimestampMixin, Base):
|
||||
source_id: Mapped[str | None] = mapped_column(String(128))
|
||||
|
||||
|
||||
class BondNominalSchedule(Base):
|
||||
"""What a bond's nominal becomes after each amortisation (plan §1.3).
|
||||
|
||||
A bond quote is a percent of the *current* nominal, not the original one, so every
|
||||
amortisation silently rewrites what a quote of «100» is worth. Without this schedule an
|
||||
amortised bond keeps being valued at its issue par and the position drifts upward by
|
||||
exactly the amount already repaid. The coupon forecast needs it for the same reason: a
|
||||
coupon is a rate on the nominal that stands on the coupon date.
|
||||
"""
|
||||
|
||||
__tablename__ = "bond_nominal_schedule"
|
||||
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
effective_date: Mapped[date] = mapped_column(primary_key=True)
|
||||
"""First date this nominal applies to."""
|
||||
nominal: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
"""moex | tinvest | ledger — `ledger` when derived from an amortisation payment."""
|
||||
|
||||
|
||||
class PositionSnapshot(Base):
|
||||
"""What the broker says a position is — kept for reconciliation, never for analytics."""
|
||||
|
||||
|
||||
@@ -0,0 +1,441 @@
|
||||
"""Which of two payout feeds to believe, and where that choice is applied (plan §Фаза 4).
|
||||
|
||||
Dividends, coupons and bond redemptions arrive from two independent feeds — T-Invest
|
||||
(`sources/tinvest/sync_events.py`) and MOEX ISS (`sources/moex/payouts.py`) — and they
|
||||
overlap. `corporate_action` is unique on `(instrument_id, kind, source, source_id)`, so both
|
||||
feeds' rows coexist in the table by construction: a row cannot displace another source's row
|
||||
even if we wanted it to.
|
||||
|
||||
**Therefore priority is a READ rule, not a write rule.** Nothing is deleted or overwritten
|
||||
when a feed syncs. The consumer — `analytics/income.py` — loads every `corporate_action` it
|
||||
cares about and calls `resolve_payouts(actions)` exactly once, which collapses the duplicates
|
||||
down to one winner per real-world payout. That placement is deliberate:
|
||||
|
||||
* a write-time rule would have to delete the loser, and the next sync of the losing source
|
||||
would insert it straight back — a table that never settles;
|
||||
* the loser is the only thing that makes the winner checkable. Keeping both rows is what
|
||||
lets `resolve_payouts` notice that the two feeds disagree on an amount and say so;
|
||||
* the priority rule itself changes as we learn the feeds. A read rule changes behaviour on
|
||||
the next refresh; a write rule would need a resync of years of history.
|
||||
|
||||
The one thing that IS decided at write time is `bond_nominal_schedule`, whose primary key is
|
||||
`(instrument_id, effective_date)` with `source` as a plain column — two feeds there really do
|
||||
collide on one row, so `nominal_outranks()` below guards that upsert.
|
||||
|
||||
## The priority itself
|
||||
|
||||
Priority is keyed on the payout's `kind`, not on the instrument's asset class, because the
|
||||
kind already says which world the payout lives in and — unlike the asset class — it is
|
||||
carried on the row itself, which is what keeps this function free of the database.
|
||||
|
||||
* **Coupons and bond redemptions: MOEX wins.** `/iss/securities/{secid}/bondization.json` is
|
||||
the issuer's own schedule as registered with the exchange: every coupon to maturity, with
|
||||
the date and the money per bond, plus the amortisation plan. T-Invest's `GetBondCoupons`
|
||||
answers for a window and in practice carries the near coupons, which is enough for a
|
||||
calendar and not enough for a forecast — and the value it reports for a floating-rate
|
||||
coupon whose rate is not yet fixed is its own estimate, not a published figure.
|
||||
* **Dividends: T-Invest wins.** `GetDividends` states the figure for the paper as the broker
|
||||
will actually settle it, including the type of the payout (`dividend_type`) — for a
|
||||
depositary receipt or a foreign issuer that is a different number from what the MOEX
|
||||
register publishes. MOEX ISS's dividend endpoint is a register extract: right for the
|
||||
ordinary share, but it knows nothing about which line of the paper is held.
|
||||
* **A fact beats an announcement.** A row with `status = paid` (or `cancelled`) outranks any
|
||||
`announced` or `forecast` row regardless of which feed it came from — that tier is checked
|
||||
BEFORE the source tier. This is what keeps the `paid` rows that
|
||||
`ledger/corporate_actions.py` derives from actual money movements from being shouted down
|
||||
by a feed's announcement of the same payout. (Those rows are also safe at write time: the
|
||||
feeds here never write `split`/`amortization`/`repayment` at all, because
|
||||
`ledger.corporate_actions._prune` owns those kinds and deletes anything in them the ledger
|
||||
does not imply. A feed's amortisation lands in `bond_nominal_schedule` instead.)
|
||||
* **A disagreement is reported, never averaged.** When the loser states a different
|
||||
`amount_per_unit` for the same `(instrument, kind, pay_date)`, the winner is still the
|
||||
winner, and the gap goes to `FINDINGS` as `payout_amount_mismatch`. A systematic ~13 % gap
|
||||
on dividends is the known one: T-Invest's `dividend_net` and the MOEX register are not
|
||||
guaranteed to be on the same side of the withholding tax.
|
||||
|
||||
`resolve_payouts` takes a plain sequence and touches no session, so `analytics/income.py`
|
||||
can call it on whatever it has already loaded, and so it is testable without a database.
|
||||
It reads only attributes (`instrument_id`, `kind`, `status`, `pay_date`, `ex_date`,
|
||||
`record_date`, `amount_per_unit`, `currency`, `source`), so any row-like object will do.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Iterable, Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS
|
||||
from fintracker.models.pricing import (
|
||||
BondNominalSchedule,
|
||||
CorporateAction,
|
||||
CorporateActionKind,
|
||||
CorporateActionStatus,
|
||||
)
|
||||
|
||||
TINVEST = "tinvest"
|
||||
MOEX = "moex"
|
||||
|
||||
#: Kinds that belong to a bond's own schedule — the ones MOEX `bondization` is authoritative on.
|
||||
BOND_KINDS = frozenset(
|
||||
{
|
||||
CorporateActionKind.coupon,
|
||||
CorporateActionKind.amortization,
|
||||
CorporateActionKind.repayment,
|
||||
CorporateActionKind.offer,
|
||||
}
|
||||
)
|
||||
|
||||
#: Feed precedence per kind family, strongest first. Anything unlisted ranks below both.
|
||||
BOND_SOURCE_ORDER = (MOEX, TINVEST)
|
||||
EQUITY_SOURCE_ORDER = (TINVEST, MOEX)
|
||||
|
||||
#: `bond_nominal_schedule` has one row per (instrument, date) whoever writes it, so this
|
||||
#: precedence is applied at write time. `ledger` sits lowest only because it is derived from
|
||||
#: a payment we saw rather than from the issuer's published plan — it is a fallback, not a lie.
|
||||
NOMINAL_SOURCE_ORDER = (MOEX, TINVEST, "ledger")
|
||||
|
||||
#: Below this the two feeds are rounding each other, not disagreeing.
|
||||
ABS_TOLERANCE = Decimal("0.0001")
|
||||
REL_TOLERANCE = Decimal("0.005")
|
||||
|
||||
_STATUS_RANK = {
|
||||
CorporateActionStatus.forecast: 0,
|
||||
CorporateActionStatus.announced: 1,
|
||||
# both are settled facts: the money moved, or the issuer called the payout off
|
||||
CorporateActionStatus.cancelled: 2,
|
||||
CorporateActionStatus.paid: 2,
|
||||
}
|
||||
|
||||
|
||||
#: The attributes a row has to carry to be resolvable. Kept as a comment rather than a
|
||||
#: `Protocol` bound on purpose: pyright matches a protocol on an attribute's DECLARED type,
|
||||
#: and every column of `CorporateAction` is declared `Mapped[...]`, so the model — the one
|
||||
#: type this function actually exists for — would fail the bound it is meant to describe.
|
||||
PAYOUT_ATTRIBUTES = (
|
||||
"instrument_id",
|
||||
"kind",
|
||||
"status",
|
||||
"record_date",
|
||||
"ex_date",
|
||||
"pay_date",
|
||||
"amount_per_unit",
|
||||
"currency",
|
||||
"source",
|
||||
)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PayoutRow:
|
||||
"""One payout as a feed produced it, before it becomes a `corporate_action` row.
|
||||
|
||||
Shared by both sources so that the two syncs cannot drift apart on what a payout is,
|
||||
and so the mapping from a feed record to this shape stays a pure function.
|
||||
"""
|
||||
|
||||
instrument_id: int
|
||||
kind: CorporateActionKind
|
||||
status: CorporateActionStatus
|
||||
source: str
|
||||
source_id: str
|
||||
record_date: date | None = None
|
||||
ex_date: date | None = None
|
||||
pay_date: date | None = None
|
||||
amount_per_unit: Decimal | None = None
|
||||
currency: str | None = None
|
||||
|
||||
def as_values(self) -> dict[str, Any]:
|
||||
"""The `corporate_action` column map for an insert."""
|
||||
return {
|
||||
"instrument_id": self.instrument_id,
|
||||
"kind": self.kind,
|
||||
"status": self.status,
|
||||
"record_date": self.record_date,
|
||||
"ex_date": self.ex_date,
|
||||
"pay_date": self.pay_date,
|
||||
"amount_per_unit": self.amount_per_unit,
|
||||
"currency": self.currency,
|
||||
"ratio": None,
|
||||
"source": self.source,
|
||||
"source_id": self.source_id,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class NominalPoint:
|
||||
"""What a bond's nominal becomes on `effective_date`, per `bond_nominal_schedule`."""
|
||||
|
||||
instrument_id: int
|
||||
effective_date: date
|
||||
nominal: Decimal
|
||||
currency: str
|
||||
source: str
|
||||
|
||||
def as_values(self) -> dict[str, Any]:
|
||||
return {
|
||||
"instrument_id": self.instrument_id,
|
||||
"effective_date": self.effective_date,
|
||||
"nominal": self.nominal,
|
||||
"currency": self.currency,
|
||||
"source": self.source,
|
||||
}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class PayoutConflict:
|
||||
"""Two feeds naming different money for one payout."""
|
||||
|
||||
instrument_id: int
|
||||
kind: CorporateActionKind
|
||||
day: date | None
|
||||
winner: Any
|
||||
loser: Any
|
||||
|
||||
@property
|
||||
def detail(self) -> str:
|
||||
return (
|
||||
f"инструмент {self.instrument_id}, {self.kind}, {self.day}: "
|
||||
f"{self.winner.source} даёт {self.winner.amount_per_unit}, "
|
||||
f"{self.loser.source} — {self.loser.amount_per_unit}"
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Resolution:
|
||||
payouts: list[Any] = field(default_factory=list)
|
||||
conflicts: list[PayoutConflict] = field(default_factory=list)
|
||||
|
||||
|
||||
def source_order(kind: Any) -> tuple[str, ...]:
|
||||
"""Feed precedence for one kind of payout, strongest first."""
|
||||
return BOND_SOURCE_ORDER if kind in BOND_KINDS else EQUITY_SOURCE_ORDER
|
||||
|
||||
|
||||
def source_rank(kind: Any, source: str) -> int:
|
||||
"""Higher is stronger. An unknown feed ranks below every known one, never above."""
|
||||
order = source_order(kind)
|
||||
return len(order) - order.index(source) if source in order else 0
|
||||
|
||||
|
||||
def status_rank(status: Any) -> int:
|
||||
return _STATUS_RANK.get(status, 0)
|
||||
|
||||
|
||||
def nominal_outranks(incoming: str, existing: str) -> bool:
|
||||
"""May a `bond_nominal_schedule` row from `incoming` replace one from `existing`?
|
||||
|
||||
Equal sources count as outranking themselves: a feed must be able to correct its own
|
||||
earlier answer, or a revised amortisation plan would never land.
|
||||
"""
|
||||
return _nominal_rank(incoming) >= _nominal_rank(existing)
|
||||
|
||||
|
||||
def weaker_or_equal_nominal_sources(incoming: str) -> list[str]:
|
||||
"""Sources an `incoming` row may overwrite — the guard list for the upsert's WHERE."""
|
||||
rank = _nominal_rank(incoming)
|
||||
return [s for s in NOMINAL_SOURCE_ORDER if _nominal_rank(s) <= rank]
|
||||
|
||||
|
||||
def _nominal_rank(source: str) -> int:
|
||||
order = NOMINAL_SOURCE_ORDER
|
||||
return len(order) - order.index(source) if source in order else 0
|
||||
|
||||
|
||||
def resolve[P: Any](actions: Iterable[P]) -> Resolution:
|
||||
"""Collapse overlapping feed rows to one payout each, pure and database-free.
|
||||
|
||||
Rows are grouped per `(instrument_id, kind)` and then per payout by `_same_payout`,
|
||||
which pairs rows that share any stated date. Within a group the winner is decided by
|
||||
the fact tier first and the feed precedence second; the losers are kept only long
|
||||
enough to check them against the winner's amount.
|
||||
"""
|
||||
result = Resolution()
|
||||
by_paper: dict[tuple[int, Any], list[P]] = {}
|
||||
for action in actions:
|
||||
by_paper.setdefault((action.instrument_id, action.kind), []).append(action)
|
||||
|
||||
for (instrument_id, kind), members in by_paper.items():
|
||||
for group in _same_payout(members):
|
||||
ranked = sorted(group, key=_rank_key, reverse=True)
|
||||
winner = ranked[0]
|
||||
result.payouts.append(winner)
|
||||
day = payout_day(winner)
|
||||
result.conflicts += [
|
||||
PayoutConflict(
|
||||
instrument_id=instrument_id, kind=kind, day=day, winner=winner, loser=loser
|
||||
)
|
||||
for loser in ranked[1:]
|
||||
if _disagrees(winner, loser)
|
||||
]
|
||||
result.payouts.sort(key=lambda a: (payout_day(a) or date.min, a.instrument_id, str(a.kind)))
|
||||
return result
|
||||
|
||||
|
||||
def _same_payout[P: Any](members: Sequence[P]) -> list[list[P]]:
|
||||
"""Split one paper's rows of one kind into groups that describe the same payout.
|
||||
|
||||
Two rows are the same payout when any of their stated dates coincide — pay, ex or
|
||||
record. Matching on a single chosen date would not work across these two feeds: MOEX's
|
||||
dividend extract states only the register-closing date while T-Invest states the record
|
||||
AND the payment date, so keying on the payment date leaves them in separate buckets and
|
||||
the income total counts the dividend twice. Two genuinely different payouts of one paper
|
||||
never share a date, which is what makes the overlap safe.
|
||||
|
||||
A row with no date at all joins nothing: without a date there is no evidence it is the
|
||||
same payout as anything else, and merging on the strength of the instrument alone would
|
||||
silently drop money.
|
||||
"""
|
||||
groups: list[tuple[set[date], list[P]]] = []
|
||||
for action in members:
|
||||
days = {d for d in (action.pay_date, action.ex_date, action.record_date) if d}
|
||||
if not days:
|
||||
groups.append((set(), [action]))
|
||||
continue
|
||||
hits = [g for g in groups if g[0] & days]
|
||||
merged_days = set(days)
|
||||
merged_rows = [action]
|
||||
for group in hits:
|
||||
merged_days |= group[0]
|
||||
merged_rows += group[1]
|
||||
groups.remove(group)
|
||||
groups.append((merged_days, merged_rows))
|
||||
return [rows for _, rows in groups]
|
||||
|
||||
|
||||
def resolve_payouts[P: Any](actions: Sequence[P], *, report: bool = True) -> list[P]:
|
||||
"""The contract `analytics/income.py` calls: one row per real payout, duplicates dropped.
|
||||
|
||||
`report=False` turns off the `FINDINGS` side effect, for callers that only want the
|
||||
resolution (a preview endpoint, a test). The resolution itself is always pure.
|
||||
"""
|
||||
result = resolve(actions)
|
||||
if report and result.conflicts:
|
||||
_report(result.conflicts)
|
||||
return result.payouts
|
||||
|
||||
|
||||
def payout_day(action: Any) -> date | None:
|
||||
"""The date two feeds can be compared on: pay date, else ex-date, else record date."""
|
||||
return action.pay_date or action.ex_date or action.record_date
|
||||
|
||||
|
||||
def _rank_key(action: Any) -> tuple[int, int, int, str]:
|
||||
"""Fact tier first, then feed precedence — a stated amount breaks a remaining tie."""
|
||||
return (
|
||||
status_rank(action.status),
|
||||
source_rank(action.kind, action.source),
|
||||
0 if action.amount_per_unit is None else 1,
|
||||
action.source or "",
|
||||
)
|
||||
|
||||
|
||||
def _disagrees(winner: Any, loser: Any) -> bool:
|
||||
"""True when both feeds named an amount and the gap is more than rounding.
|
||||
|
||||
A missing amount on either side is a gap, not a disagreement: one feed simply has not
|
||||
published the figure yet (a floating coupon whose rate is unfixed), and calling that a
|
||||
conflict would fill the quality report with noise on every refresh.
|
||||
"""
|
||||
a, b = winner.amount_per_unit, loser.amount_per_unit
|
||||
if a is None or b is None:
|
||||
return False
|
||||
if (winner.currency or "") != (loser.currency or ""):
|
||||
return True
|
||||
return abs(a - b) > max(ABS_TOLERANCE, abs(a) * REL_TOLERANCE)
|
||||
|
||||
|
||||
def _report(conflicts: Sequence[PayoutConflict]) -> None:
|
||||
FINDINGS.add(
|
||||
"payout_amount_mismatch",
|
||||
"warn",
|
||||
f"Источники расходятся в сумме выплаты по {len(conflicts)} записям "
|
||||
f"(взята запись по приоритету): {'; '.join(c.detail for c in conflicts[:5])}",
|
||||
count=len(conflicts),
|
||||
ref={"instruments": sorted({c.instrument_id for c in conflicts})},
|
||||
)
|
||||
|
||||
|
||||
async def upsert_payouts(session: AsyncSession, rows: Sequence[PayoutRow]) -> int:
|
||||
"""Write feed payouts idempotently on `(instrument_id, kind, source, source_id)`.
|
||||
|
||||
Every feed goes through here, so nothing else has to remember that a re-sync of the same
|
||||
window must be a no-op, or that a row belonging to another `source` is never touched.
|
||||
"""
|
||||
if not rows:
|
||||
return 0
|
||||
# one statement cannot address the same key twice; the last mapping of a key wins
|
||||
unique = {(r.instrument_id, r.kind, r.source, r.source_id): r for r in rows}
|
||||
stmt = pg_insert(CorporateAction).values([r.as_values() for r in unique.values()])
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["instrument_id", "kind", "source", "source_id"],
|
||||
set_={
|
||||
"status": stmt.excluded.status,
|
||||
"record_date": stmt.excluded.record_date,
|
||||
"ex_date": stmt.excluded.ex_date,
|
||||
"pay_date": stmt.excluded.pay_date,
|
||||
"amount_per_unit": stmt.excluded.amount_per_unit,
|
||||
"currency": stmt.excluded.currency,
|
||||
},
|
||||
)
|
||||
)
|
||||
return len(unique)
|
||||
|
||||
|
||||
async def upsert_nominals(session: AsyncSession, points: Sequence[NominalPoint]) -> int:
|
||||
"""Write a bond's nominal schedule, letting only an equal-or-stronger source overwrite.
|
||||
|
||||
This is the one place a payout feed decides a precedence at write time, and it has to:
|
||||
`bond_nominal_schedule` is keyed on `(instrument_id, effective_date)` with `source` as a
|
||||
plain column, so MOEX's published amortisation plan and T-Invest's redemption events land
|
||||
on the very same row. See `nominal_outranks`.
|
||||
"""
|
||||
if not points:
|
||||
return 0
|
||||
unique = {(p.instrument_id, p.effective_date): p for p in points}
|
||||
written = 0
|
||||
by_source: dict[str, list[NominalPoint]] = {}
|
||||
for point in unique.values():
|
||||
by_source.setdefault(point.source, []).append(point)
|
||||
for source, batch in by_source.items():
|
||||
stmt = pg_insert(BondNominalSchedule).values([p.as_values() for p in batch])
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["instrument_id", "effective_date"],
|
||||
set_={
|
||||
"nominal": stmt.excluded.nominal,
|
||||
"currency": stmt.excluded.currency,
|
||||
"source": stmt.excluded.source,
|
||||
},
|
||||
where=BondNominalSchedule.source.in_(weaker_or_equal_nominal_sources(source)),
|
||||
)
|
||||
)
|
||||
written += len(batch)
|
||||
return written
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BOND_KINDS",
|
||||
"MOEX",
|
||||
"NOMINAL_SOURCE_ORDER",
|
||||
"TINVEST",
|
||||
"NominalPoint",
|
||||
"PayoutConflict",
|
||||
"PayoutRow",
|
||||
"Resolution",
|
||||
"nominal_outranks",
|
||||
"payout_day",
|
||||
"resolve",
|
||||
"resolve_payouts",
|
||||
"source_order",
|
||||
"source_rank",
|
||||
"status_rank",
|
||||
"upsert_nominals",
|
||||
"upsert_payouts",
|
||||
"weaker_or_equal_nominal_sources",
|
||||
]
|
||||
@@ -1,8 +1,10 @@
|
||||
"""The `moex` source: prices and bond schedules from the MOEX ISS."""
|
||||
|
||||
from fintracker.sources.moex.payouts import MoexPayoutsSource
|
||||
from fintracker.sources.moex.sync import MoexSource
|
||||
from fintracker.sources.registry import register
|
||||
|
||||
register(MoexSource())
|
||||
register(MoexPayoutsSource())
|
||||
|
||||
__all__ = ["MoexSource"]
|
||||
__all__ = ["MoexPayoutsSource", "MoexSource"]
|
||||
|
||||
@@ -0,0 +1,354 @@
|
||||
"""The `moex_payouts` source: the second payout feed, from MOEX ISS (plan §Фаза 4).
|
||||
|
||||
Two endpoints, both free and keyless:
|
||||
|
||||
* `/iss/securities/{secid}/bondization.json` — the bond's registered schedule: every coupon
|
||||
to maturity and the amortisation plan. `MoexClient.bondization` already reads it.
|
||||
* `/iss/securities/{secid}/dividends.json` — the dividend register extract for a share.
|
||||
Only this module uses it, so the request lives here rather than on the client.
|
||||
|
||||
What it writes, and what it deliberately does not:
|
||||
|
||||
* coupons -> `corporate_action(kind=coupon, source='moex')`, alongside the T-Invest rows
|
||||
for the same coupons rather than instead of them. Which one an analytic reads is decided
|
||||
by `pricing/payouts.resolve_payouts`, at read time — see that module for why.
|
||||
* dividends -> `corporate_action(kind=dividend, source='moex')`, same arrangement.
|
||||
* amortisations -> `bond_nominal_schedule(source='moex')`, and NOT
|
||||
`corporate_action(kind=amortization)`: that kind belongs to `ledger/corporate_actions.py`,
|
||||
whose prune deletes every row in it the ledger does not imply.
|
||||
|
||||
**The amortisation plan is read as a run-out, not as a column.** ISS states `value` (repaid
|
||||
per bond) and `facevalue` per row, but which side of the payment `facevalue` stands on is not
|
||||
documented and differs between papers. Summing what is still to be repaid is unambiguous:
|
||||
everything the issuer will ever repay per bond is the nominal, so the nominal standing after
|
||||
a given amortisation is the sum of those after it. `facevalue` is used only as a cross-check,
|
||||
and a mismatch is a warning rather than a different answer.
|
||||
|
||||
**There is no raw tier here.** `raw_moex_doc` (plan §1.1) does not exist yet, and the MOEX
|
||||
source has never had one: ISS answers are cheap to re-fetch, unlike the rate-limited
|
||||
T-Invest feeds the raw tables exist for.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import httpx
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import today_local
|
||||
from fintracker.models import AssetClass, Event, EventStatus, Instrument
|
||||
from fintracker.models.pricing import CorporateActionKind, CorporateActionStatus
|
||||
from fintracker.pricing.payouts import (
|
||||
MOEX,
|
||||
NominalPoint,
|
||||
PayoutRow,
|
||||
upsert_nominals,
|
||||
upsert_payouts,
|
||||
)
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.moex.client import (
|
||||
BASE,
|
||||
AmortisationRow,
|
||||
CouponRow,
|
||||
MoexClient,
|
||||
MoexError,
|
||||
_currency,
|
||||
_date,
|
||||
_decimal,
|
||||
_rows,
|
||||
new_http_client,
|
||||
)
|
||||
from fintracker.sources.moex.sync import _secid_candidates
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = MOEX
|
||||
NAME = "moex_payouts"
|
||||
|
||||
BOND_CLASSES = frozenset({AssetClass.bond})
|
||||
DIVIDEND_CLASSES = frozenset({AssetClass.share, AssetClass.etf, AssetClass.fund})
|
||||
|
||||
ZERO = Decimal(0)
|
||||
NOMINAL_TOLERANCE = Decimal("0.01")
|
||||
"""Below this the run-out and the stated face value are rounding each other."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MoexDividendRow:
|
||||
"""One row of `/iss/securities/{secid}/dividends.json`.
|
||||
|
||||
The extract is a register snapshot: it states when the register closed and how much per
|
||||
share was declared, and nothing about when the money is actually paid — which is the
|
||||
gap T-Invest fills, and one of the reasons dividends are read from T-Invest first.
|
||||
"""
|
||||
|
||||
secid: str
|
||||
registry_close_date: date | None
|
||||
value: Decimal | None
|
||||
currency: str | None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Target:
|
||||
instrument_id: int
|
||||
secid: str
|
||||
asset_class: AssetClass
|
||||
currency: str
|
||||
|
||||
|
||||
async def fetch_dividends(client: httpx.AsyncClient, secid: str) -> list[MoexDividendRow]:
|
||||
"""The dividend register extract for one paper; an unlisted paper answers with nothing."""
|
||||
response = await client.get(
|
||||
f"{BASE}/securities/{secid}/dividends.json", params={"iss.meta": "off"}
|
||||
)
|
||||
if response.status_code == httpx.codes.NOT_FOUND:
|
||||
raise MoexError(f"dividends for {secid} not found")
|
||||
response.raise_for_status()
|
||||
return [
|
||||
MoexDividendRow(
|
||||
secid=str(row.get("secid") or secid),
|
||||
registry_close_date=_date(row.get("registryclosedate")),
|
||||
value=_decimal(row.get("value")),
|
||||
currency=_currency(row.get("currencyid")),
|
||||
)
|
||||
for row in _rows(response.json(), "dividends")
|
||||
]
|
||||
|
||||
|
||||
def coupon_payout(row: CouponRow, *, instrument_id: int, today: date) -> PayoutRow | None:
|
||||
"""One bondization coupon as a `corporate_action` row.
|
||||
|
||||
A coupon whose rate is not fixed yet arrives dated but with no `value`; it is kept, with
|
||||
a NULL amount, because the date is real and the calendar needs it. `resolve_payouts`
|
||||
treats a missing amount as a gap rather than a disagreement for exactly this case.
|
||||
"""
|
||||
if row.coupon_date is None:
|
||||
return None
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.coupon,
|
||||
status=_status(row.coupon_date, today),
|
||||
source=SOURCE,
|
||||
source_id=f"cpn:{row.coupon_date.isoformat()}",
|
||||
pay_date=row.coupon_date,
|
||||
amount_per_unit=row.value,
|
||||
currency=(row.currency or "").upper() or None,
|
||||
)
|
||||
|
||||
|
||||
def dividend_payout(row: MoexDividendRow, *, instrument_id: int, today: date) -> PayoutRow | None:
|
||||
"""One register row as a `corporate_action` row.
|
||||
|
||||
The register-closing date is the record date, and it is the only date stated — so it is
|
||||
also what a date-based merge with the T-Invest row keys on.
|
||||
"""
|
||||
if row.registry_close_date is None:
|
||||
return None
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.dividend,
|
||||
status=_status(row.registry_close_date, today),
|
||||
source=SOURCE,
|
||||
source_id=f"div:{row.registry_close_date.isoformat()}",
|
||||
record_date=row.registry_close_date,
|
||||
amount_per_unit=row.value,
|
||||
currency=(row.currency or "").upper() or None,
|
||||
)
|
||||
|
||||
|
||||
def nominal_schedule(
|
||||
amortisations: Sequence[AmortisationRow],
|
||||
*,
|
||||
instrument_id: int,
|
||||
currency: str,
|
||||
warnings: list[str],
|
||||
secid: str = "",
|
||||
) -> list[NominalPoint]:
|
||||
"""The nominal standing after each amortisation, run out from the total still to repay."""
|
||||
usable = sorted(
|
||||
(a for a in amortisations if a.amort_date is not None and a.value is not None),
|
||||
key=lambda a: a.amort_date or date.min,
|
||||
)
|
||||
if not usable:
|
||||
return []
|
||||
|
||||
total = sum((a.value or ZERO for a in usable), start=ZERO)
|
||||
stated = max((a.face_value for a in usable if a.face_value is not None), default=None)
|
||||
if stated is not None and abs(stated - total) > NOMINAL_TOLERANCE:
|
||||
warnings.append(
|
||||
f"{secid}: сумма амортизаций {total} расходится с номиналом {stated} "
|
||||
"— график построен по сумме выплат"
|
||||
)
|
||||
|
||||
remaining = total
|
||||
points: list[NominalPoint] = []
|
||||
for row in usable:
|
||||
remaining -= row.value or ZERO
|
||||
points.append(
|
||||
NominalPoint(
|
||||
instrument_id=instrument_id,
|
||||
effective_date=row.amort_date or date.min,
|
||||
nominal=remaining,
|
||||
currency=(row.currency or currency).upper(),
|
||||
source=SOURCE,
|
||||
)
|
||||
)
|
||||
return points
|
||||
|
||||
|
||||
def _status(day: date, today: date) -> CorporateActionStatus:
|
||||
"""Past dates are facts the issuer has settled; future ones are announcements."""
|
||||
return CorporateActionStatus.paid if day < today else CorporateActionStatus.announced
|
||||
|
||||
|
||||
class MoexPayoutsSource:
|
||||
"""`Source` protocol, same shape as `MoexSource`. Registration is done elsewhere."""
|
||||
|
||||
name = NAME
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
session = ctx.session
|
||||
today = today_local()
|
||||
targets = await load_targets(session)
|
||||
if not targets:
|
||||
log.info("moex_payouts: no priceable instruments in the ledger yet")
|
||||
return SyncResult(cursor_after=today.isoformat(), counts={"payouts": 0}, changed=False)
|
||||
|
||||
counts = {"instruments": 0, "coupons": 0, "dividends": 0, "payouts": 0, "nominals": 0}
|
||||
warnings: list[str] = []
|
||||
payouts: list[PayoutRow] = []
|
||||
nominals: list[NominalPoint] = []
|
||||
|
||||
async with new_http_client() as http, MoexClient(http) as moex:
|
||||
for target in targets:
|
||||
counts["instruments"] += 1
|
||||
if target.asset_class in BOND_CLASSES:
|
||||
rows = await self._bond(moex, target, today, payouts, nominals, warnings)
|
||||
counts["coupons"] += rows
|
||||
else:
|
||||
counts["dividends"] += await self._dividends(
|
||||
http, target, today, payouts, warnings
|
||||
)
|
||||
|
||||
counts["payouts"] = await upsert_payouts(session, payouts)
|
||||
counts["nominals"] = await upsert_nominals(session, nominals)
|
||||
await session.commit()
|
||||
|
||||
log.info(
|
||||
"moex_payouts: %s instruments, %s payouts, %s nominal points",
|
||||
counts["instruments"],
|
||||
counts["payouts"],
|
||||
counts["nominals"],
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=bool(counts["payouts"] or counts["nominals"]),
|
||||
)
|
||||
|
||||
async def _bond(
|
||||
self,
|
||||
moex: MoexClient,
|
||||
target: Target,
|
||||
today: date,
|
||||
payouts: list[PayoutRow],
|
||||
nominals: list[NominalPoint],
|
||||
warnings: list[str],
|
||||
) -> int:
|
||||
coupons: list[CouponRow] = []
|
||||
amortisations: list[AmortisationRow] = []
|
||||
for secid in _secid_candidates(target.secid):
|
||||
try:
|
||||
coupons, amortisations = await moex.bondization(secid)
|
||||
except (MoexError, httpx.HTTPError) as err:
|
||||
warnings.append(f"{secid}: {err}")
|
||||
continue
|
||||
if coupons or amortisations:
|
||||
break
|
||||
payouts += [
|
||||
payout
|
||||
for coupon in coupons
|
||||
if (payout := coupon_payout(coupon, instrument_id=target.instrument_id, today=today))
|
||||
]
|
||||
nominals += nominal_schedule(
|
||||
amortisations,
|
||||
instrument_id=target.instrument_id,
|
||||
currency=target.currency,
|
||||
warnings=warnings,
|
||||
secid=target.secid,
|
||||
)
|
||||
return len(coupons)
|
||||
|
||||
async def _dividends(
|
||||
self,
|
||||
http: httpx.AsyncClient,
|
||||
target: Target,
|
||||
today: date,
|
||||
payouts: list[PayoutRow],
|
||||
warnings: list[str],
|
||||
) -> int:
|
||||
rows: list[MoexDividendRow] = []
|
||||
for secid in _secid_candidates(target.secid):
|
||||
try:
|
||||
rows = await fetch_dividends(http, secid)
|
||||
except (MoexError, httpx.HTTPError) as err:
|
||||
warnings.append(f"{secid}: {err}")
|
||||
continue
|
||||
if rows:
|
||||
break
|
||||
payouts += [
|
||||
payout
|
||||
for row in rows
|
||||
if (payout := dividend_payout(row, instrument_id=target.instrument_id, today=today))
|
||||
]
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def load_targets(session: AsyncSession) -> list[Target]:
|
||||
"""Instruments the confirmed ledger touches that MOEX can answer about at all."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Instrument.id,
|
||||
Instrument.ticker,
|
||||
Instrument.asset_class,
|
||||
Instrument.currency,
|
||||
)
|
||||
.join(Event, Event.instrument_id == Instrument.id)
|
||||
.where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Instrument.ticker.is_not(None),
|
||||
Instrument.asset_class.in_(BOND_CLASSES | DIVIDEND_CLASSES),
|
||||
)
|
||||
.group_by(Instrument.id, Instrument.ticker, Instrument.asset_class, Instrument.currency)
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
Target(
|
||||
instrument_id=r.id,
|
||||
secid=r.ticker,
|
||||
asset_class=r.asset_class,
|
||||
currency=r.currency,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
__all__ = [
|
||||
"NAME",
|
||||
"MoexDividendRow",
|
||||
"MoexPayoutsSource",
|
||||
"Target",
|
||||
"coupon_payout",
|
||||
"dividend_payout",
|
||||
"fetch_dividends",
|
||||
"load_targets",
|
||||
"nominal_schedule",
|
||||
]
|
||||
@@ -2,7 +2,9 @@
|
||||
|
||||
from fintracker.sources.registry import register
|
||||
from fintracker.sources.tinvest.sync import TinvestSource
|
||||
from fintracker.sources.tinvest.sync_events import TinvestEventsSource
|
||||
|
||||
register(TinvestSource())
|
||||
register(TinvestEventsSource())
|
||||
|
||||
__all__ = ["TinvestSource"]
|
||||
__all__ = ["TinvestEventsSource", "TinvestSource"]
|
||||
|
||||
@@ -155,6 +155,61 @@ class InstrumentInfo:
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class DividendRow:
|
||||
"""One `Dividend` from GetDividends, flattened.
|
||||
|
||||
`instrument_uid` is the uid we ASKED for: the record itself carries no instrument id
|
||||
at all, so the only link back to the paper is the request.
|
||||
"""
|
||||
|
||||
instrument_uid: str
|
||||
amount: Decimal | None
|
||||
"""`dividend_net` — the per-share figure T-Invest publishes."""
|
||||
currency: str | None
|
||||
payment_date: datetime | None
|
||||
declared_date: datetime | None
|
||||
record_date: datetime | None
|
||||
last_buy_date: datetime | None
|
||||
"""Last day a purchase still earns the dividend; the ex-date is the next trading day."""
|
||||
dividend_type: str | None
|
||||
regularity: str | None
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BondCouponRow:
|
||||
"""One `Coupon` from GetBondCoupons."""
|
||||
|
||||
instrument_uid: str
|
||||
coupon_number: int | None
|
||||
coupon_date: datetime | None
|
||||
fix_date: datetime | None
|
||||
pay_one_bond: Decimal | None
|
||||
currency: str | None
|
||||
coupon_type: str
|
||||
"""The enum's NAME, e.g. COUPON_TYPE_CONSTANT — an unknown one is the caller's call."""
|
||||
coupon_period: int | None
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class BondEventRow:
|
||||
"""One `BondEvent` from GetBondEvents (coupons, calls, redemptions)."""
|
||||
|
||||
instrument_uid: str
|
||||
event_type: str
|
||||
"""The enum's NAME: EVENT_TYPE_CPN | EVENT_TYPE_CALL | EVENT_TYPE_MTY | EVENT_TYPE_CONV."""
|
||||
event_number: int | None
|
||||
event_date: datetime | None
|
||||
fix_date: datetime | None
|
||||
pay_date: datetime | None
|
||||
pay_one_bond: Decimal | None
|
||||
"""Money paid per bond — for a redemption event this is the principal repaid."""
|
||||
currency: str | None
|
||||
payload: dict[str, Any]
|
||||
|
||||
|
||||
def _as_dict(message: Any) -> dict[str, Any]:
|
||||
"""A JSON-able dict for the `raw_*` tables.
|
||||
|
||||
@@ -436,6 +491,125 @@ class TinvestClient:
|
||||
country=info.country or match.country,
|
||||
)
|
||||
|
||||
async def dividends(
|
||||
self, instrument_uid: str, *, since: datetime, until: datetime
|
||||
) -> list[DividendRow]:
|
||||
"""GetDividends over [since, until] for one paper.
|
||||
|
||||
Only shares and depositary receipts have a dividend history here; asking about a
|
||||
bond, an ETF or a currency answers NOT_FOUND or an empty list, which is a fact
|
||||
about the paper rather than an error — hence the empty list instead of a raise.
|
||||
"""
|
||||
resp = await self._payout_call(
|
||||
"GetDividends",
|
||||
instrument_uid,
|
||||
lambda: self._client.instruments.get_dividends(
|
||||
instrument_id=instrument_uid, from_=since, to=until
|
||||
),
|
||||
)
|
||||
if resp is None:
|
||||
return []
|
||||
return [
|
||||
DividendRow(
|
||||
instrument_uid=instrument_uid,
|
||||
amount=_money(d.dividend_net),
|
||||
currency=_currency(d.dividend_net),
|
||||
payment_date=getattr(d, "payment_date", None),
|
||||
declared_date=getattr(d, "declared_date", None),
|
||||
record_date=getattr(d, "record_date", None),
|
||||
last_buy_date=getattr(d, "last_buy_date", None),
|
||||
dividend_type=getattr(d, "dividend_type", None) or None,
|
||||
regularity=getattr(d, "regularity", None) or None,
|
||||
payload=_as_dict(d),
|
||||
)
|
||||
for d in resp.dividends
|
||||
]
|
||||
|
||||
async def bond_coupons(
|
||||
self, instrument_uid: str, *, since: datetime, until: datetime
|
||||
) -> list[BondCouponRow]:
|
||||
"""GetBondCoupons: the coupon schedule T-Invest holds for a bond."""
|
||||
resp = await self._payout_call(
|
||||
"GetBondCoupons",
|
||||
instrument_uid,
|
||||
lambda: self._client.instruments.get_bond_coupons(
|
||||
instrument_id=instrument_uid, from_=since, to=until
|
||||
),
|
||||
)
|
||||
if resp is None:
|
||||
return []
|
||||
return [
|
||||
BondCouponRow(
|
||||
instrument_uid=instrument_uid,
|
||||
coupon_number=int(getattr(c, "coupon_number", 0) or 0) or None,
|
||||
coupon_date=getattr(c, "coupon_date", None),
|
||||
fix_date=getattr(c, "fix_date", None),
|
||||
pay_one_bond=_money(c.pay_one_bond),
|
||||
currency=_currency(c.pay_one_bond),
|
||||
coupon_type=getattr(getattr(c, "coupon_type", None), "name", "") or "",
|
||||
coupon_period=int(getattr(c, "coupon_period", 0) or 0) or None,
|
||||
payload=_as_dict(c),
|
||||
)
|
||||
for c in resp.events
|
||||
]
|
||||
|
||||
async def bond_events(
|
||||
self, instrument_uid: str, *, since: datetime, until: datetime, event_type: str
|
||||
) -> list[BondEventRow]:
|
||||
"""GetBondEvents of one `EventType` name (EVENT_TYPE_CPN, EVENT_TYPE_MTY, …).
|
||||
|
||||
The request takes exactly one type, so a caller after both coupons and redemptions
|
||||
pays two RPCs. Amortisation has no type of its own: a partially amortised bond
|
||||
reports several `EVENT_TYPE_MTY` events, each repaying a slice of the principal.
|
||||
"""
|
||||
# `t_tech.invest` re-exports neither of these — they live only in `.schemas`
|
||||
from t_tech.invest.schemas import EventType, GetBondEventsRequest
|
||||
|
||||
try:
|
||||
kind = EventType[event_type]
|
||||
except KeyError:
|
||||
raise ValueError(f"unknown bond EventType: {event_type}") from None
|
||||
resp = await self._payout_call(
|
||||
"GetBondEvents",
|
||||
instrument_uid,
|
||||
lambda: self._client.instruments.get_bond_events(
|
||||
GetBondEventsRequest(instrument_id=instrument_uid, from_=since, to=until, type=kind)
|
||||
),
|
||||
)
|
||||
if resp is None:
|
||||
return []
|
||||
return [
|
||||
BondEventRow(
|
||||
instrument_uid=instrument_uid,
|
||||
event_type=getattr(getattr(e, "event_type", None), "name", "") or "",
|
||||
event_number=int(getattr(e, "event_number", 0) or 0) or None,
|
||||
event_date=getattr(e, "event_date", None),
|
||||
fix_date=getattr(e, "fix_date", None),
|
||||
pay_date=getattr(e, "pay_date", None) or getattr(e, "real_pay_date", None),
|
||||
pay_one_bond=_money(e.pay_one_bond),
|
||||
currency=_currency(e.pay_one_bond),
|
||||
payload=_as_dict(e),
|
||||
)
|
||||
for e in resp.events
|
||||
]
|
||||
|
||||
async def _payout_call(
|
||||
self, rpc: str, instrument_uid: str, fn: Callable[[], Coroutine[Any, Any, T]]
|
||||
) -> T | None:
|
||||
"""`_call`, but a paper the RPC does not serve yields None instead of failing the run.
|
||||
|
||||
The payout RPCs are typed to an asset class: a share has no coupon schedule and an
|
||||
ETF has no dividend history in this feed, and both answer NOT_FOUND or
|
||||
INVALID_ARGUMENT. One such paper must not abort a sync over the whole portfolio.
|
||||
"""
|
||||
try:
|
||||
return await self._call(fn)
|
||||
except AioRequestError as err:
|
||||
if err.code in (StatusCode.NOT_FOUND, StatusCode.INVALID_ARGUMENT):
|
||||
log.info("tinvest: %s has no %s data", instrument_uid, rpc)
|
||||
return None
|
||||
raise
|
||||
|
||||
|
||||
def _silence_sdk_telemetry() -> None:
|
||||
"""Stop the SDK from reporting our errors to T-Bank's Sentry.
|
||||
|
||||
@@ -0,0 +1,463 @@
|
||||
"""The `tinvest_events` source: the payout calendar as T-Invest publishes it (plan §Фаза 4).
|
||||
|
||||
One run, per instrument the ledger has ever touched:
|
||||
|
||||
1. shares, ETFs -> `GetDividends` -> `raw_tinvest_event('dividend')` -> `corporate_action`
|
||||
2. bonds -> `GetBondCoupons` -> `raw_tinvest_event('coupon')` -> `corporate_action`
|
||||
3. bonds -> `GetBondEvents` -> `raw_tinvest_event('bond_event')` -> `bond_nominal_schedule`
|
||||
|
||||
**Scope comes from the ledger, not from the current portfolio.** A paper that was sold last
|
||||
year still paid dividends while it was held, and the income history has to keep them.
|
||||
|
||||
**The window is the paper's whole life, not an increment.** These feeds are schedules, not
|
||||
streams: a coupon plan is revised in place (a floating rate gets fixed, a date moves), and an
|
||||
incremental read would keep an obsolete row forever. The cost is bounded — one RPC per share,
|
||||
two per bond, on a portfolio of under a hundred papers — and `client._call` waits out the
|
||||
Instruments limit of 200/min on its own, so a full sweep is slow rather than fragile.
|
||||
|
||||
**Amortisation does not go into `corporate_action`.** `ledger/corporate_actions.py` owns the
|
||||
kinds `split`, `amortization` and `repayment` end to end: its `_prune` deletes every row in
|
||||
those kinds the ledger does not imply, so anything this module wrote there would survive
|
||||
until the next refresh and no longer. What a feed knows that the ledger cannot is the nominal
|
||||
the bond carries *before* the money arrives, and that is exactly what
|
||||
`bond_nominal_schedule` is for — the amortisation cash is the difference between two
|
||||
consecutive nominals. Dividends and coupons are untouched by that prune and are written
|
||||
normally, as `announced` ahead of the pay date and `paid` once it has passed.
|
||||
|
||||
**Redemption events carry no nominal, only money.** `GetBondEvents(EVENT_TYPE_MTY)` returns
|
||||
one event per partial redemption with `pay_one_bond` — the slice of principal repaid. The
|
||||
schedule is reconstructed by running that backwards from the total: everything the issuer
|
||||
will ever repay per bond IS the original nominal, so the nominal standing after each
|
||||
redemption is the sum of the redemptions still to come. Reading `instrument.nominal` instead
|
||||
would not work: T-Invest reports the *current* nominal, which is already amortised down.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from collections.abc import Sequence
|
||||
from datetime import UTC, date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from sqlalchemy import select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import today_local
|
||||
from fintracker.models import AssetClass, Event, EventStatus, Instrument, RawTinvestEvent
|
||||
from fintracker.models.pricing import CorporateActionKind, CorporateActionStatus
|
||||
from fintracker.pricing.payouts import (
|
||||
TINVEST,
|
||||
NominalPoint,
|
||||
PayoutRow,
|
||||
upsert_nominals,
|
||||
upsert_payouts,
|
||||
)
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.tinvest.client import (
|
||||
BondCouponRow,
|
||||
BondEventRow,
|
||||
DividendRow,
|
||||
TinvestClient,
|
||||
)
|
||||
from fintracker.sources.tinvest.sync import TinvestAuthError
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = TINVEST
|
||||
NAME = "tinvest_events"
|
||||
MSK = ZoneInfo("Europe/Moscow")
|
||||
|
||||
HISTORY_START = datetime(2015, 1, 1, tzinfo=UTC)
|
||||
"""Far enough back for any paper the portfolio has held; the API clamps to the issue date."""
|
||||
FORWARD_YEARS = 10
|
||||
"""How far ahead to ask. A coupon plan runs to maturity, and a long OFZ is a decade out."""
|
||||
|
||||
#: Which asset classes have a dividend history in this feed at all.
|
||||
DIVIDEND_CLASSES = frozenset({AssetClass.share, AssetClass.etf, AssetClass.fund})
|
||||
BOND_CLASSES = frozenset({AssetClass.bond})
|
||||
|
||||
#: Coupon types the API states. `UNSPECIFIED` is not one of them — it is the API declining
|
||||
#: to say, and a payout whose nature is unknown is warned about, never filed as "other".
|
||||
KNOWN_COUPON_TYPES = frozenset(
|
||||
{
|
||||
"COUPON_TYPE_CONSTANT",
|
||||
"COUPON_TYPE_FLOATING",
|
||||
"COUPON_TYPE_DISCOUNT",
|
||||
"COUPON_TYPE_MORTGAGE",
|
||||
"COUPON_TYPE_FIX",
|
||||
"COUPON_TYPE_VARIABLE",
|
||||
"COUPON_TYPE_OTHER",
|
||||
}
|
||||
)
|
||||
|
||||
REDEMPTION = "EVENT_TYPE_MTY"
|
||||
"""The only bond event type this sync asks for: partial and final redemptions."""
|
||||
|
||||
ZERO = Decimal(0)
|
||||
|
||||
|
||||
class Target:
|
||||
"""One instrument to ask about, with the identity and shape the feeds need."""
|
||||
|
||||
__slots__ = ("asset_class", "currency", "instrument_id", "ticker", "uid")
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
instrument_id: int,
|
||||
uid: str,
|
||||
asset_class: AssetClass,
|
||||
currency: str,
|
||||
ticker: str | None,
|
||||
) -> None:
|
||||
self.instrument_id = instrument_id
|
||||
self.uid = uid
|
||||
self.asset_class = asset_class
|
||||
self.currency = currency
|
||||
self.ticker = ticker
|
||||
|
||||
|
||||
def msk_date(value: datetime | None) -> date | None:
|
||||
"""A feed timestamp as the trading day it belongs to (conventions: trade dates in MSK).
|
||||
|
||||
The API stamps these at midnight UTC, which is the previous evening in Moscow — reading
|
||||
the date off the UTC value moves every payout one day earlier.
|
||||
"""
|
||||
if value is None:
|
||||
return None
|
||||
if value.tzinfo is None:
|
||||
value = value.replace(tzinfo=UTC)
|
||||
return value.astimezone(MSK).date()
|
||||
|
||||
|
||||
def status_for(day: date | None, today: date) -> CorporateActionStatus:
|
||||
"""A payout whose date has passed is `paid`; one still ahead is `announced`.
|
||||
|
||||
The feed states the issuer's schedule, not our bank statement, so `paid` here means "the
|
||||
issuer paid it", which is what the calendar and the income forecast need. Whether the
|
||||
money reached a specific account is the ledger's answer, and the ledger's `paid` rows
|
||||
outrank these — see `pricing/payouts.py`.
|
||||
"""
|
||||
if day is not None and day < today:
|
||||
return CorporateActionStatus.paid
|
||||
return CorporateActionStatus.announced
|
||||
|
||||
|
||||
def dividend_payout(row: DividendRow, *, instrument_id: int, today: date) -> PayoutRow | None:
|
||||
"""One `GetDividends` record as a `corporate_action` row, or None when it says nothing.
|
||||
|
||||
`last_buy_date` is the last day a purchase still earns the dividend, which is the day
|
||||
BEFORE the ex-date — but it is the only ex-side date the feed states, and storing it as
|
||||
`ex_date` keeps the calendar honest to within one trading day. The record date is the
|
||||
one that decides entitlement, and it is stated exactly.
|
||||
"""
|
||||
record = msk_date(row.record_date)
|
||||
pay = msk_date(row.payment_date)
|
||||
declared = msk_date(row.declared_date)
|
||||
key = record or pay or declared
|
||||
if key is None:
|
||||
return None
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.dividend,
|
||||
status=status_for(pay or record, today),
|
||||
source=SOURCE,
|
||||
source_id=f"div:{key.isoformat()}",
|
||||
record_date=record,
|
||||
ex_date=msk_date(row.last_buy_date),
|
||||
pay_date=pay,
|
||||
amount_per_unit=row.amount,
|
||||
currency=row.currency,
|
||||
)
|
||||
|
||||
|
||||
def coupon_payout(
|
||||
row: BondCouponRow, *, instrument_id: int, today: date, warnings: list[str]
|
||||
) -> PayoutRow | None:
|
||||
"""One `GetBondCoupons` record as a `corporate_action` row.
|
||||
|
||||
A zero payment is dropped without a warning: that is a discount bond's nominal coupon,
|
||||
not a payout. An unrecognised `coupon_type` IS warned about and dropped — filing it as
|
||||
a plain coupon would put money of an unknown nature into the income forecast.
|
||||
"""
|
||||
if row.coupon_type not in KNOWN_COUPON_TYPES:
|
||||
warnings.append(
|
||||
f"{row.instrument_uid}: незнакомый тип купона {row.coupon_type or '<пусто>'} — пропущен"
|
||||
)
|
||||
return None
|
||||
day = msk_date(row.coupon_date)
|
||||
if day is None:
|
||||
return None
|
||||
if row.pay_one_bond is not None and row.pay_one_bond == ZERO:
|
||||
return None
|
||||
key = row.coupon_number if row.coupon_number else day.isoformat()
|
||||
return PayoutRow(
|
||||
instrument_id=instrument_id,
|
||||
kind=CorporateActionKind.coupon,
|
||||
status=status_for(day, today),
|
||||
source=SOURCE,
|
||||
source_id=f"cpn:{key}",
|
||||
record_date=msk_date(row.fix_date),
|
||||
ex_date=None,
|
||||
pay_date=day,
|
||||
amount_per_unit=row.pay_one_bond,
|
||||
currency=row.currency,
|
||||
)
|
||||
|
||||
|
||||
def nominal_schedule(
|
||||
events: Sequence[BondEventRow],
|
||||
*,
|
||||
instrument_id: int,
|
||||
currency: str,
|
||||
warnings: list[str],
|
||||
) -> list[NominalPoint]:
|
||||
"""The nominal standing after each redemption, from the redemptions themselves.
|
||||
|
||||
Everything the issuer repays per bond over its life is the original nominal, so the
|
||||
nominal left after a given redemption is the sum of those still ahead of it. The final
|
||||
entry is therefore zero on the maturity date, which is the truth: the paper is gone.
|
||||
|
||||
A redemption with no money attached cannot be placed in the run and is warned about
|
||||
rather than treated as zero — a silent zero would shift every later nominal upward.
|
||||
"""
|
||||
usable: list[tuple[date, Decimal, str | None]] = []
|
||||
for event in events:
|
||||
if event.event_type != REDEMPTION:
|
||||
warnings.append(
|
||||
f"{event.instrument_uid}: незнакомый тип события облигации "
|
||||
f"{event.event_type or '<пусто>'} — пропущено"
|
||||
)
|
||||
continue
|
||||
day = msk_date(event.pay_date) or msk_date(event.event_date)
|
||||
if day is None:
|
||||
continue
|
||||
if event.pay_one_bond is None:
|
||||
warnings.append(f"{event.instrument_uid}: погашение {day} без суммы — пропущено")
|
||||
continue
|
||||
usable.append((day, event.pay_one_bond, event.currency))
|
||||
|
||||
if not usable:
|
||||
return []
|
||||
usable.sort()
|
||||
remaining = sum((amount for _, amount, _ in usable), start=ZERO)
|
||||
points: list[NominalPoint] = []
|
||||
for day, amount, ccy in usable:
|
||||
remaining -= amount
|
||||
points.append(
|
||||
NominalPoint(
|
||||
instrument_id=instrument_id,
|
||||
effective_date=day,
|
||||
nominal=remaining,
|
||||
currency=(ccy or currency).upper(),
|
||||
source=SOURCE,
|
||||
)
|
||||
)
|
||||
return points
|
||||
|
||||
|
||||
class TinvestEventsSource:
|
||||
"""`Source` protocol, same shape as `MoexSource`. Registration is done elsewhere."""
|
||||
|
||||
name = NAME
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
token = ctx.settings.tinvest_token
|
||||
if not token:
|
||||
raise TinvestAuthError(
|
||||
"TINVEST_TOKEN is not set — put a T-Invest token in .env "
|
||||
"(t-bank.ru -> Инвестиции -> настройки -> токены)."
|
||||
)
|
||||
|
||||
session = ctx.session
|
||||
today = today_local()
|
||||
targets = await load_targets(session)
|
||||
if not targets:
|
||||
log.info("tinvest_events: no T-Invest instruments in the ledger yet")
|
||||
return SyncResult(cursor_after=today.isoformat(), counts={"payouts": 0}, changed=False)
|
||||
|
||||
until = datetime(today.year + FORWARD_YEARS, 12, 31, tzinfo=UTC)
|
||||
counts = {"instruments": 0, "dividends": 0, "coupons": 0, "redemptions": 0, "nominals": 0}
|
||||
warnings: list[str] = []
|
||||
payouts: list[PayoutRow] = []
|
||||
nominals: list[NominalPoint] = []
|
||||
raw: list[dict[str, Any]] = []
|
||||
|
||||
async with TinvestClient(token) as client:
|
||||
for target in targets:
|
||||
counts["instruments"] += 1
|
||||
if target.asset_class in DIVIDEND_CLASSES:
|
||||
rows = await client.dividends(target.uid, since=HISTORY_START, until=until)
|
||||
counts["dividends"] += len(rows)
|
||||
for row in rows:
|
||||
payout = dividend_payout(
|
||||
row, instrument_id=target.instrument_id, today=today
|
||||
)
|
||||
if payout is None:
|
||||
continue
|
||||
payouts.append(payout)
|
||||
raw.append(
|
||||
_raw(
|
||||
target.uid,
|
||||
"dividend",
|
||||
payout.source_id,
|
||||
payout.record_date,
|
||||
row.payload,
|
||||
)
|
||||
)
|
||||
elif target.asset_class in BOND_CLASSES:
|
||||
coupons = await client.bond_coupons(
|
||||
target.uid, since=HISTORY_START, until=until
|
||||
)
|
||||
counts["coupons"] += len(coupons)
|
||||
for coupon in coupons:
|
||||
payout = coupon_payout(
|
||||
coupon,
|
||||
instrument_id=target.instrument_id,
|
||||
today=today,
|
||||
warnings=warnings,
|
||||
)
|
||||
if payout is None:
|
||||
continue
|
||||
payouts.append(payout)
|
||||
raw.append(
|
||||
_raw(
|
||||
target.uid,
|
||||
"coupon",
|
||||
payout.source_id,
|
||||
payout.pay_date,
|
||||
coupon.payload,
|
||||
)
|
||||
)
|
||||
|
||||
events = await client.bond_events(
|
||||
target.uid, since=HISTORY_START, until=until, event_type=REDEMPTION
|
||||
)
|
||||
counts["redemptions"] += len(events)
|
||||
points = nominal_schedule(
|
||||
events,
|
||||
instrument_id=target.instrument_id,
|
||||
currency=target.currency,
|
||||
warnings=warnings,
|
||||
)
|
||||
nominals += points
|
||||
for index, event in enumerate(events):
|
||||
day = msk_date(event.pay_date) or msk_date(event.event_date)
|
||||
raw.append(
|
||||
_raw(
|
||||
target.uid,
|
||||
"bond_event",
|
||||
f"mty:{day if day else index}",
|
||||
day,
|
||||
event.payload,
|
||||
)
|
||||
)
|
||||
|
||||
await _store_raw(session, raw)
|
||||
written = await upsert_payouts(session, payouts)
|
||||
counts["nominals"] = await upsert_nominals(session, nominals)
|
||||
counts["payouts"] = written
|
||||
await session.commit()
|
||||
|
||||
log.info(
|
||||
"tinvest_events: %s instruments, %s payouts, %s nominal points",
|
||||
counts["instruments"],
|
||||
counts["payouts"],
|
||||
counts["nominals"],
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=bool(written or counts["nominals"]),
|
||||
)
|
||||
|
||||
|
||||
async def load_targets(session: AsyncSession) -> list[Target]:
|
||||
"""Every T-Invest instrument the confirmed ledger touches — held now or held once."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Instrument.id,
|
||||
Instrument.tinvest_uid,
|
||||
Instrument.asset_class,
|
||||
Instrument.currency,
|
||||
Instrument.ticker,
|
||||
)
|
||||
.join(Event, Event.instrument_id == Instrument.id)
|
||||
.where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Instrument.tinvest_uid.is_not(None),
|
||||
Instrument.asset_class.in_(DIVIDEND_CLASSES | BOND_CLASSES),
|
||||
)
|
||||
.group_by(
|
||||
Instrument.id,
|
||||
Instrument.tinvest_uid,
|
||||
Instrument.asset_class,
|
||||
Instrument.currency,
|
||||
Instrument.ticker,
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return [
|
||||
Target(
|
||||
instrument_id=r.id,
|
||||
uid=r.tinvest_uid,
|
||||
asset_class=r.asset_class,
|
||||
currency=r.currency,
|
||||
ticker=r.ticker,
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
def _raw(
|
||||
uid: str, kind: str, source_id: str, day: date | None, payload: dict[str, Any]
|
||||
) -> dict[str, Any]:
|
||||
return {
|
||||
"instrument_uid": uid,
|
||||
"kind": kind,
|
||||
"source_id": source_id,
|
||||
"event_date": day,
|
||||
"payload": payload,
|
||||
}
|
||||
|
||||
|
||||
async def _store_raw(session: AsyncSession, rows: Sequence[dict[str, Any]]) -> None:
|
||||
"""Append-only, idempotent on `(instrument_uid, kind, source_id)` — the raw-tier rule.
|
||||
|
||||
The payload is refreshed rather than kept at its first version: these feeds revise a
|
||||
schedule in place, and the point of the raw tier is to be able to re-derive the current
|
||||
mapping, not to keep a history of what the API used to say.
|
||||
"""
|
||||
if not rows:
|
||||
return
|
||||
unique = {(r["instrument_uid"], r["kind"], r["source_id"]): r for r in rows}
|
||||
stmt = pg_insert(RawTinvestEvent).values(list(unique.values()))
|
||||
await session.execute(
|
||||
stmt.on_conflict_do_update(
|
||||
index_elements=["instrument_uid", "kind", "source_id"],
|
||||
set_={
|
||||
"event_date": stmt.excluded.event_date,
|
||||
"payload": stmt.excluded.payload,
|
||||
"fetched_at": datetime.now(UTC),
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"KNOWN_COUPON_TYPES",
|
||||
"NAME",
|
||||
"REDEMPTION",
|
||||
"Target",
|
||||
"TinvestEventsSource",
|
||||
"coupon_payout",
|
||||
"dividend_payout",
|
||||
"load_targets",
|
||||
"msk_date",
|
||||
"nominal_schedule",
|
||||
"status_for",
|
||||
]
|
||||
Reference in New Issue
Block a user