diff --git a/backend/alembic/versions/2bf84b07fd5e_фаза_4_доходы_ребалансировка_бенчмарки_.py b/backend/alembic/versions/2bf84b07fd5e_фаза_4_доходы_ребалансировка_бенчмарки_.py new file mode 100644 index 0000000..337e833 --- /dev/null +++ b/backend/alembic/versions/2bf84b07fd5e_фаза_4_доходы_ребалансировка_бенчмарки_.py @@ -0,0 +1,417 @@ +"""фаза 4: доходы, ребалансировка, бенчмарки, цели, налоги + +Revision ID: 2bf84b07fd5e +Revises: ee170b3e7872 +Create Date: 2026-09-18 18:41:12.207755 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op +from sqlalchemy.dialects import postgresql + +# `allocation_dimension` already exists (фаза 2, metric_allocation). Declaring it with +# `create_type=False` reuses it instead of re-issuing CREATE TYPE, which would fail on any +# re-upgrade — the type outlives a downgrade because `metric_allocation` still uses it. + + +revision: str = "2bf84b07fd5e" +down_revision: str | None = "ee170b3e7872" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + + +def upgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "goal", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("name", sa.String(length=128), nullable=False), + sa.Column("scope", sa.String(length=32), nullable=False), + sa.Column("target_amount", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("currency", sa.String(length=3), nullable=False), + sa.Column("target_date", sa.Date(), nullable=True), + sa.Column("monthly_contribution", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("note", sa.Text(), nullable=True), + sa.Column("archived", sa.Boolean(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_goal")), + sa.UniqueConstraint("name", name=op.f("uq_goal_name")), + ) + op.create_table( + "metric_income_monthly", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("scope", sa.String(length=32), nullable=False), + sa.Column("month", sa.Date(), nullable=False), + sa.Column("kind", sa.String(length=16), nullable=False), + sa.Column("currency", sa.String(length=3), nullable=False), + sa.Column("amount", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("amount_rub", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("tax_withheld", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("payment_count", sa.Integer(), nullable=False), + sa.Column( + "computed_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_income_monthly")), + sa.UniqueConstraint( + "scope", + "month", + "kind", + "currency", + name=op.f("uq_metric_income_monthly_scope_month_kind_currency"), + ), + ) + op.create_index( + op.f("ix_metric_income_monthly_scope"), "metric_income_monthly", ["scope"], unique=False + ) + op.create_table( + "benchmark", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("code", sa.String(length=32), nullable=False), + sa.Column("name", sa.String(length=128), nullable=False), + sa.Column("kind", sa.Enum("price", "total_return", name="benchmark_kind"), nullable=False), + sa.Column("instrument_id", sa.Integer(), nullable=True), + sa.Column("source", sa.String(length=16), nullable=False), + sa.Column("currency", sa.String(length=3), nullable=False), + sa.Column("is_default", sa.Boolean(), nullable=False), + sa.Column("is_active", sa.Boolean(), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["instrument_id"], + ["instrument.id"], + name=op.f("fk_benchmark_instrument_id_instrument"), + ondelete="SET NULL", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_benchmark")), + sa.UniqueConstraint("code", name=op.f("uq_benchmark_code")), + ) + op.create_table( + "bond_nominal_schedule", + sa.Column("instrument_id", sa.Integer(), nullable=False), + sa.Column("effective_date", sa.Date(), nullable=False), + sa.Column("nominal", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("currency", sa.String(length=3), nullable=False), + sa.Column("source", sa.String(length=16), nullable=False), + sa.ForeignKeyConstraint( + ["instrument_id"], + ["instrument.id"], + name=op.f("fk_bond_nominal_schedule_instrument_id_instrument"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint( + "instrument_id", "effective_date", name=op.f("pk_bond_nominal_schedule") + ), + ) + op.create_table( + "metric_goal_progress", + sa.Column("goal_id", sa.Integer(), nullable=False), + sa.Column("as_of", sa.Date(), nullable=False), + sa.Column("current_value_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("target_amount_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("progress", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("projected_date", sa.Date(), nullable=True), + sa.Column("basis", sa.String(length=16), nullable=False), + sa.Column("assumed_rate", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("monthly_needed_rub", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("on_track", sa.Boolean(), nullable=True), + sa.Column( + "computed_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["goal_id"], + ["goal.id"], + name=op.f("fk_metric_goal_progress_goal_id_goal"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("goal_id", name=op.f("pk_metric_goal_progress")), + ) + op.create_table( + "metric_income_calendar", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("scope", sa.String(length=32), nullable=False), + sa.Column("instrument_id", sa.Integer(), nullable=False), + sa.Column("kind", sa.String(length=16), nullable=False), + sa.Column("expected_date", sa.Date(), nullable=False), + sa.Column("record_date", sa.Date(), nullable=True), + sa.Column("qty", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("per_unit", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("amount", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("currency", sa.String(length=3), nullable=False), + sa.Column("amount_rub", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column( + "basis", + sa.Enum("schedule", "announced", "history", "paid", name="income_basis"), + nullable=False, + ), + sa.Column("tax_withheld", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column( + "computed_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["instrument_id"], + ["instrument.id"], + name=op.f("fk_metric_income_calendar_instrument_id_instrument"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_income_calendar")), + sa.UniqueConstraint( + "scope", + "instrument_id", + "kind", + "expected_date", + "basis", + name=op.f("uq_metric_income_calendar_scope_instrument_id_kind_expected_date_basis"), + ), + ) + op.create_index( + op.f("ix_metric_income_calendar_expected_date"), + "metric_income_calendar", + ["expected_date"], + unique=False, + ) + op.create_index( + op.f("ix_metric_income_calendar_scope"), "metric_income_calendar", ["scope"], unique=False + ) + op.create_table( + "metric_rebalance", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("portfolio_id", sa.Integer(), nullable=False), + sa.Column( + "dimension", + postgresql.ENUM( + "asset_class", + "sector", + "country", + "currency", + name="allocation_dimension", + create_type=False, + ), + nullable=False, + ), + sa.Column("bucket", sa.String(length=64), nullable=False), + sa.Column("instrument_id", sa.Integer(), nullable=True), + sa.Column("current_value_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("current_weight", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("target_weight", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("delta_value_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("suggested_qty", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("lot", sa.Integer(), nullable=True), + sa.Column("price", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("price_currency", sa.String(length=3), nullable=True), + sa.Column("within_band", sa.Boolean(), nullable=False), + sa.Column("blocked_by_cash", sa.Boolean(), nullable=False), + sa.Column( + "computed_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["instrument_id"], + ["instrument.id"], + name=op.f("fk_metric_rebalance_instrument_id_instrument"), + ondelete="CASCADE", + ), + sa.ForeignKeyConstraint( + ["portfolio_id"], + ["portfolio.id"], + name=op.f("fk_metric_rebalance_portfolio_id_portfolio"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_rebalance")), + sa.UniqueConstraint( + "portfolio_id", + "dimension", + "bucket", + "instrument_id", + name=op.f("uq_metric_rebalance_portfolio_id_dimension_bucket_instrument_id"), + ), + ) + op.create_table( + "metric_tax_year", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("year", sa.Integer(), nullable=False), + sa.Column("account_id", sa.Integer(), nullable=False), + sa.Column("dividends_gross_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("coupons_gross_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("tax_withheld_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("realized_gain_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("realized_loss_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("ldv_exempt_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("taxable_base_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("estimated_tax_rub", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("tax_rate", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column( + "computed_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["account_id"], + ["account.id"], + name=op.f("fk_metric_tax_year_account_id_account"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_tax_year")), + sa.UniqueConstraint("year", "account_id", name=op.f("uq_metric_tax_year_year_account_id")), + ) + op.create_index(op.f("ix_metric_tax_year_year"), "metric_tax_year", ["year"], unique=False) + op.create_table( + "portfolio_target", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("portfolio_id", sa.Integer(), nullable=False), + sa.Column( + "dimension", + postgresql.ENUM( + "asset_class", + "sector", + "country", + "currency", + name="allocation_dimension", + create_type=False, + ), + nullable=False, + ), + sa.Column("bucket", sa.String(length=64), nullable=False), + sa.Column("target_weight", sa.Numeric(precision=24, scale=10), nullable=False), + sa.Column("band", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("note", sa.Text(), nullable=True), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["portfolio_id"], + ["portfolio.id"], + name=op.f("fk_portfolio_target_portfolio_id_portfolio"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_portfolio_target")), + sa.UniqueConstraint( + "portfolio_id", + "dimension", + "bucket", + name=op.f("uq_portfolio_target_portfolio_id_dimension_bucket"), + ), + ) + op.create_index( + op.f("ix_portfolio_target_portfolio_id"), "portfolio_target", ["portfolio_id"], unique=False + ) + op.create_table( + "metric_benchmark_returns", + sa.Column("id", sa.Integer(), nullable=False), + sa.Column("scope", sa.String(length=32), nullable=False), + sa.Column("period", sa.String(length=8), nullable=False), + sa.Column("benchmark_id", sa.Integer(), nullable=False), + sa.Column("date_from", sa.Date(), nullable=False), + sa.Column("date_to", sa.Date(), nullable=False), + sa.Column("twr", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("twr_annualized", sa.Numeric(precision=24, scale=10), nullable=True), + sa.Column("days_skipped", sa.Integer(), nullable=False), + sa.Column( + "computed_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=False, + ), + sa.ForeignKeyConstraint( + ["benchmark_id"], + ["benchmark.id"], + name=op.f("fk_metric_benchmark_returns_benchmark_id_benchmark"), + ondelete="CASCADE", + ), + sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_benchmark_returns")), + sa.UniqueConstraint( + "scope", + "period", + "benchmark_id", + name=op.f("uq_metric_benchmark_returns_scope_period_benchmark_id"), + ), + ) + op.create_index( + op.f("ix_metric_benchmark_returns_scope"), + "metric_benchmark_returns", + ["scope"], + unique=False, + ) + op.add_column( + "metric_allocation", + sa.Column("target_weight", sa.Numeric(precision=24, scale=10), nullable=True), + ) + op.add_column( + "metric_allocation", sa.Column("drift", sa.Numeric(precision=24, scale=10), nullable=True) + ) + # ### end Alembic commands ### + + +def downgrade() -> None: + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("metric_allocation", "drift") + op.drop_column("metric_allocation", "target_weight") + op.drop_index(op.f("ix_metric_benchmark_returns_scope"), table_name="metric_benchmark_returns") + op.drop_table("metric_benchmark_returns") + op.drop_index(op.f("ix_portfolio_target_portfolio_id"), table_name="portfolio_target") + op.drop_table("portfolio_target") + op.drop_index(op.f("ix_metric_tax_year_year"), table_name="metric_tax_year") + op.drop_table("metric_tax_year") + op.drop_table("metric_rebalance") + op.drop_index(op.f("ix_metric_income_calendar_scope"), table_name="metric_income_calendar") + op.drop_index( + op.f("ix_metric_income_calendar_expected_date"), table_name="metric_income_calendar" + ) + op.drop_table("metric_income_calendar") + op.drop_table("metric_goal_progress") + op.drop_table("bond_nominal_schedule") + op.drop_table("benchmark") + op.drop_index(op.f("ix_metric_income_monthly_scope"), table_name="metric_income_monthly") + op.drop_table("metric_income_monthly") + op.drop_table("goal") + # ### end Alembic commands ### + + # Dropping a table leaves its enum type behind, and a re-upgrade would then fail. + # `allocation_dimension` stays: `metric_allocation` still uses it. + sa.Enum(name="income_basis").drop(op.get_bind(), checkfirst=True) + sa.Enum(name="benchmark_kind").drop(op.get_bind(), checkfirst=True) diff --git a/backend/src/fintracker/analytics/__init__.py b/backend/src/fintracker/analytics/__init__.py index 5812f0e..f4f33b0 100644 --- a/backend/src/fintracker/analytics/__init__.py +++ b/backend/src/fintracker/analytics/__init__.py @@ -103,14 +103,18 @@ def register_steps() -> None: from fintracker.analytics import ( allocation, + benchmarks, cashflow, cashflow_broker, classify, + income, networth, quality, + rebalance, returns, runway, spending, + tax, valuation, ) from fintracker.ledger.corporate_actions import rebuild_corporate_actions @@ -132,8 +136,15 @@ def register_steps() -> None: # valuation prices the positions the lots describe; returns reads the series it writes register_step("valuation", valuation.rebuild_valuation) register_step("returns", returns.rebuild_returns) + # benchmarks reads metric_returns to sit on the same date grid + register_step("benchmarks", benchmarks.rebuild_benchmark_returns) register_step("allocation", allocation.rebuild_allocation) + # rebalance reads metric_allocation, never re-derives a weight itself + register_step("rebalance", rebalance.rebuild_rebalance) register_step("cashflow_broker", cashflow_broker.rebuild_cash_flow_broker) + # income and tax both read event/corporate_action/lot_disposal, all written by now + register_step("income", income.rebuild_income) + register_step("tax", tax.rebuild_tax_year) register_step("networth", networth.rebuild_net_worth_daily) register_step("cashflow", cashflow.rebuild_cash_flow_monthly) register_step("spending", spending.rebuild_spending_by_category) diff --git a/backend/src/fintracker/analytics/benchmarks.py b/backend/src/fintracker/analytics/benchmarks.py new file mode 100644 index 0000000..63edb04 --- /dev/null +++ b/backend/src/fintracker/analytics/benchmarks.py @@ -0,0 +1,256 @@ +"""A benchmark's time-weighted return, on the portfolio's own grid of days (plan §4, фаза 4). + +An index quoted on MOEX is an ordinary `instrument`: `sources/moex` fetches its history from +the ISS endpoint the same way it fetches a share's, and it lands in `price_daily`. Nothing +is seeded from code — the list of benchmarks is data in the `benchmark` table, because which +index a portfolio is measured against is the user's decision. The codes expected there are +MOEX secids: + +* ``IMOEX`` — the price index of the Moscow Exchange, ``kind = "price"``; +* ``MCFTR`` — its gross total-return twin, ``kind = "total_return"``; +* ``RGBITR`` — the government-bond total-return index, for a bond-heavy portfolio. + +**Why the kind matters more than it looks.** IMOEX drops by the dividend on every ex-date +and never gets it back, so over a decade it trails MCFTR by roughly the dividend yield +compounded — for the Russian market, several percent a year. A holder of the same basket +received those dividends. Comparing a portfolio's TWR (which includes the income it earned) +against a price index therefore flatters the portfolio by construction, every single year, +with no market view behind it. MCFTR is the honest opponent: both sides count dividends. +This module does not "fix" a price index by adding a dividend estimate — that would be an +invented number. It reports `kind` outward so the client can mark the comparison for what it +is. + +**Why the grid must be shared.** A return is a product of daily factors, and a factor only +exists for a day both sides have a value for. If the index is chained over its own trading +days while the portfolio is chained over every calendar day, the two numbers answer +different questions and their difference is not an excess return. So the days come from +`metric_portfolio_value_daily` for the same scope, over the same `date_from`/`date_to` that +`metric_returns` recorded, and a day the index has no quote for is **counted**, not hidden: +`days_skipped` is reported on both sides and the client must show it when it is non-zero. + +A skipped day does not lose the market move. The previous quote stays as the base, so the +next quoted day links back to it and the chain still telescopes to `p_end / p_start`. What +`days_skipped` says is narrower and exactly true: on that many days of the compared window +the index had nothing to say, so the two series are not day-for-day comparable. + +There are no external flows in an index, which is why this is much simpler than +`analytics/returns.py`: the TWR of an index is just its price change. +""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + +from sqlalchemy import delete, insert, select +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import FINDINGS +from fintracker.analytics.returns import annualize +from fintracker.models import ( + Benchmark, + MetricBenchmarkReturns, + MetricPortfolioValueDaily, + MetricReturns, + PriceDaily, +) + +log = logging.getLogger(__name__) + +ZERO = Decimal(0) +ONE = Decimal(1) +RATE_PLACES = Decimal("0.000001") + + +@dataclass(frozen=True) +class IndexChain: + """An index's cumulative return and how much of the compared window it covers.""" + + value: Decimal | None + days_used: int + days_skipped: int + + +# -------------------------------------------------------------------------------------- +# pure core +# -------------------------------------------------------------------------------------- + + +def opening_price(prices: Mapping[date, Decimal], opening: date) -> Decimal | None: + """The index's level at the start of the period: its last quote on or before `opening`. + + The first day of a period is a closing value, and periods start on a calendar date that + is often a weekend — `1y` back from a Sunday is a Sunday. Refusing to start there would + leave a whole period unmeasured over a detail of the calendar, so the last known close is + used. It is not a substituted number: it is the level the index actually stood at. + """ + if opening in prices: + return prices[opening] + earlier = [d for d in prices if d < opening] + return prices[max(earlier)] if earlier else None + + +def index_twr( + prices: Mapping[date, Decimal], grid: Sequence[date], *, opening: Decimal | None +) -> IndexChain: + """Chain the index's daily price factors over exactly the days in `grid`. + + `grid` is the portfolio's own set of days inside the period (the opening day excluded, + as in `returns.twr`). A day with no quote contributes no factor and increments + `days_skipped`; the base price is kept, so the move that eventually shows up is credited + to the first quoted day after the gap rather than lost. + """ + if opening is None or opening <= ZERO: + return IndexChain(None, 0, len(grid)) + factor = ONE + previous = opening + used = skipped = 0 + for d in grid: + price = prices.get(d) + if price is None: + skipped += 1 + continue + if previous > ZERO: + factor *= price / previous + used += 1 + previous = price + if not used: + return IndexChain(None, 0, skipped) + return IndexChain((factor - ONE).quantize(RATE_PLACES), used, skipped) + + +# -------------------------------------------------------------------------------------- +# I/O +# -------------------------------------------------------------------------------------- + + +async def rebuild_benchmark_returns(session: AsyncSession) -> None: + """Replace `metric_benchmark_returns` for every active benchmark and every metric row.""" + await session.execute(delete(MetricBenchmarkReturns)) + + found = await session.execute( + select(Benchmark).where(Benchmark.is_active.is_(True)).order_by(Benchmark.id) + ) + benchmarks = list(found.scalars().all()) + if not benchmarks: + return + + priced = [b for b in benchmarks if b.instrument_id is not None] + silent = [b.code for b in benchmarks if b.instrument_id is None] + if silent: + FINDINGS.add( + "benchmark_no_instrument", + "warn", + "Бенчмарки без привязанного инструмента, истории нет: " + ", ".join(sorted(silent)), + count=len(silent), + ) + if not priced: + return + + prices = await _load_prices(session, {b.instrument_id for b in priced if b.instrument_id}) + grids = await _load_grids(session) + periods = ( + await session.execute( + select( + MetricReturns.scope, + MetricReturns.period, + MetricReturns.date_from, + MetricReturns.date_to, + ) + ) + ).all() + + rows: list[dict[str, object]] = [] + skipped_total = 0 + for p in periods: + grid = [d for d in grids.get(p.scope, ()) if p.date_from < d <= p.date_to] + if not grid: + continue + for benchmark in priced: + series = prices.get(benchmark.instrument_id or 0, {}) + chain = index_twr( + series, grid, opening=opening_price(series, p.date_from) if series else None + ) + skipped_total += chain.days_skipped + rows.append( + { + "scope": p.scope, + "period": p.period, + "benchmark_id": benchmark.id, + "date_from": p.date_from, + "date_to": p.date_to, + "twr": chain.value, + # annualised over the CALENDAR span, not over the quoted days: an index + # trades ~250 days a year, so scaling by its own day count would treat a + # full year as eight months and inflate every long-period number + "twr_annualized": annualize(chain.value, (p.date_to - p.date_from).days), + "days_skipped": chain.days_skipped, + } + ) + + if rows: + await session.execute(insert(MetricBenchmarkReturns), rows) + _report(rows, skipped_total) + log.info("benchmarks: %s rows over %s benchmarks", len(rows), len(priced)) + + +async def _load_prices( + session: AsyncSession, instrument_ids: set[int] +) -> dict[int, dict[date, Decimal]]: + """Each index's closes by day, exactly as quoted — no carry-forward. + + Carrying a close forward here would erase the very thing `days_skipped` exists to report: + a holiday would silently become a day with a return of zero. + """ + if not instrument_ids: + return {} + rows = ( + await session.execute( + select(PriceDaily.instrument_id, PriceDaily.d, PriceDaily.close).where( + PriceDaily.instrument_id.in_(instrument_ids) + ) + ) + ).all() + out: dict[int, dict[date, Decimal]] = defaultdict(dict) + for r in rows: + out[r.instrument_id][r.d] = Decimal(r.close) + return dict(out) + + +async def _load_grids(session: AsyncSession) -> dict[str, list[date]]: + """The days each scope was valued on — the same spine `analytics/returns.py` chained.""" + rows = ( + await session.execute( + select(MetricPortfolioValueDaily.scope, MetricPortfolioValueDaily.d).order_by( + MetricPortfolioValueDaily.scope, MetricPortfolioValueDaily.d + ) + ) + ).all() + out: dict[str, list[date]] = defaultdict(list) + for r in rows: + out[r.scope].append(r.d) + return dict(out) + + +def _report(rows: Sequence[Mapping[str, object]], skipped_total: int) -> None: + """Say out loud when a comparison is not day-for-day.""" + empty = sum(1 for r in rows if r["twr"] is None) + if empty: + FINDINGS.add( + "benchmark_no_history", + "warn", + f"У бенчмарков нет котировок на {empty} сравнений — сравнить не с чем", + count=empty, + ) + if skipped_total: + FINDINGS.add( + "benchmark_days_skipped", + "info", + f"Дней без котировки индекса в сравниваемых окнах: {skipped_total} — " + "сетка портфеля и индекса совпадает не полностью", + count=skipped_total, + ) diff --git a/backend/src/fintracker/analytics/goals.py b/backend/src/fintracker/analytics/goals.py new file mode 100644 index 0000000..a57fed7 --- /dev/null +++ b/backend/src/fintracker/analytics/goals.py @@ -0,0 +1,436 @@ +"""How far a goal has come, and when it gets there (plan §Фаза 4). + +A goal is an amount, a scope and optionally a date. Everything on the screen is derived: +where the scope stands today (`metric_portfolio_value_daily`), what it has been earning +(`metric_returns`), and what the user said they would keep adding +(`goal.monthly_contribution`). + +**The projected date is allowed to be NULL, and often should be.** A portfolio that is flat +or falling and receives nothing new never reaches a target above its current value. The +honest output for that is «not on this trend», not a date in 2087 — a far date looks like an +answer without being one, and a client cannot tell the two apart. So the projection runs a +month at a time over a bounded horizon (`MAX_HORIZON_MONTHS`) and returns NULL if the target +is still out of reach at the end of it. + +**Which rate is assumed, and when a rate is assumed at all** — that is what `basis` names: + +* `xirr` — the scope's own trailing money-weighted return, taken from `metric_returns`. + Only from a period spanning at least `MIN_XIRR_HISTORY_DAYS` (180): an annualised rate is + a per-year figure, and a goal projection compounds it over years, so a rate extracted from + six weeks of a single position moving would be multiplied into a decade of fiction. (The + 30-day floor `analytics/returns.py` uses for a per-instrument XIRR is a display threshold; + this one feeds a forecast, which is a stronger claim.) Among the qualifying periods the + **shortest** is preferred, because a goal is about what the portfolio is doing now, not in + 2019 — `1y` before `3y` before `all`. +* `contribution` — no usable trailing return, but the user plans to add money every month. + The projection then assumes zero growth: what is planned is a deposit, not a yield. +* `none` — neither. There is no trend to extrapolate, so `projected_date` is NULL. + +A negative trailing XIRR is used as it is, not clamped to zero. If the goal also receives +contributions it may still be reached, slower; if it does not, the projection returns NULL, +which is the requirement. + +**`monthly_needed_rub`** answers the other direction: given the deadline, the current value +and the assumed rate, how much has to go in every month. NULL without a deadline, and NULL +once the deadline has passed — there is no «per month» in zero months left. +""" + +from __future__ import annotations + +import logging +from collections.abc import Mapping, Sequence +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + +from sqlalchemy import delete, insert, select +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import FINDINGS, today_local +from fintracker.models import ( + Goal, + MetricGoalProgress, + MetricPortfolioValueDaily, + MetricReturns, +) +from fintracker.pricing.fx import FxTable + +log = logging.getLogger(__name__) + +ZERO = Decimal(0) +ONE = Decimal(1) +MONTHS_IN_YEAR = Decimal(12) +RATE_PLACES = Decimal("0.000001") +MONEY_PLACES = Decimal("0.01") + +MIN_XIRR_HISTORY_DAYS = 180 +"""Shortest trailing window whose annualised return may be compounded into a forecast.""" + +MAX_HORIZON_MONTHS = 360 +"""30 years. Past this the projection stops being information and starts being a shrug, so +the answer becomes NULL instead of a date nobody would plan around.""" + +#: Preference order among the periods `metric_returns` stores: the recent trend first. +PERIOD_PREFERENCE = ("1y", "3y", "all", "ytd", "6m", "3m", "1m") + +BASIS_XIRR = "xirr" +BASIS_CONTRIBUTION = "contribution" +BASIS_NONE = "none" + + +@dataclass(frozen=True) +class Projection: + projected_date: date | None + basis: str + assumed_rate: Decimal | None + + +@dataclass(frozen=True) +class GoalProgress: + goal_id: int + as_of: date + current_value_rub: Decimal + target_amount_rub: Decimal + progress: Decimal + projected_date: date | None + basis: str + assumed_rate: Decimal | None + monthly_needed_rub: Decimal | None + on_track: bool | None + + +def monthly_rate(annual: Decimal) -> Decimal: + """The monthly rate that compounds to `annual` over twelve months. + + Decimal all the way: `(1 + r) ** (1/12)` is exactly what `decimal` computes correctly + rounded, so no float ever touches a number that ends up in a stored column. + """ + base = ONE + annual + if base <= ZERO: + # a loss of 100 % or worse has no real twelfth root; treat it as total loss + return Decimal(-1) + return base ** (ONE / MONTHS_IN_YEAR) - ONE + + +def add_months(d: date, months: int) -> date: + total = d.year * 12 + (d.month - 1) + months + year, month = divmod(total, 12) + day = min(d.day, _days_in_month(year, month + 1)) + return date(year, month + 1, day) + + +def _days_in_month(year: int, month: int) -> int: + if month == 12: + return 31 + return (date(year, month + 1, 1) - date(year, month, 1)).days + + +def months_between(start: date, end: date) -> int: + """Whole months from `start` to `end`, never negative.""" + months = (end.year - start.year) * 12 + (end.month - start.month) + if end.day < start.day: + months -= 1 + return max(0, months) + + +def project( + *, + current: Decimal, + target: Decimal, + as_of: date, + annual_rate: Decimal | None, + monthly_contribution: Decimal, + basis: str, +) -> Projection: + """Step the balance forward a month at a time until it reaches the target, or give up. + + Iteration rather than a closed-form solve because the closed form needs a logarithm (a + float) and has to special-case a zero rate, a negative rate and a contribution that + exactly offsets the drawdown. A loop of at most 360 Decimal multiplications is cheap, + exact, and impossible to read wrong. + """ + if basis == BASIS_NONE: + return Projection(None, BASIS_NONE, None) + if current >= target: + return Projection(as_of, basis, annual_rate) + + rate = annual_rate if annual_rate is not None else ZERO + step = monthly_rate(rate) + value = current + for month in range(1, MAX_HORIZON_MONTHS + 1): + value = value * (ONE + step) + monthly_contribution + if value >= target: + return Projection(add_months(as_of, month), basis, annual_rate) + return Projection(None, basis, annual_rate) + + +def monthly_needed( + *, current: Decimal, target: Decimal, months: int, annual_rate: Decimal | None +) -> Decimal | None: + """Contribution per month that lands exactly on `target` after `months` months. + + Same iteration argument as `project`: the annuity formula divides by the rate and breaks + at zero, so the required payment is found by bisection on a function that is monotone in + the payment. Returns 0 when the goal is already met on the trend alone. + """ + if months <= 0: + return None + rate = annual_rate if annual_rate is not None else ZERO + step = monthly_rate(rate) + + def final(payment: Decimal) -> Decimal: + value = current + for _ in range(months): + value = value * (ONE + step) + payment + return value + + if final(ZERO) >= target: + return ZERO + low, high = ZERO, max(target, ONE) + while final(high) < target: + high *= 2 + if high > target * 1000: + return None + for _ in range(60): + mid = (low + high) / 2 + if final(mid) < target: + low = mid + else: + high = mid + return high.quantize(MONEY_PLACES) + + +def pick_rate(rows: Sequence[Mapping[str, object]]) -> tuple[Decimal | None, str]: + """Trailing XIRR to project with, and the period it came from. + + `rows` are `metric_returns` rows of the goal's scope as mappings with `period`, + `date_from`, `date_to` and `xirr`. + """ + usable = {} + for row in rows: + xirr = row.get("xirr") + date_from, date_to = row.get("date_from"), row.get("date_to") + if xirr is None or not isinstance(date_from, date) or not isinstance(date_to, date): + continue + if (date_to - date_from).days < MIN_XIRR_HISTORY_DAYS: + continue + usable[str(row.get("period"))] = Decimal(str(xirr)) + for period in PERIOD_PREFERENCE: + if period in usable: + return usable[period], period + return None, "" + + +def evaluate( + *, + goal_id: int, + as_of: date, + current: Decimal, + target: Decimal, + target_date: date | None, + monthly_contribution: Decimal | None, + trailing_xirr: Decimal | None, +) -> GoalProgress: + """Everything the progress row holds, from numbers already in hand.""" + contribution = monthly_contribution or ZERO + if trailing_xirr is not None: + basis, rate = BASIS_XIRR, trailing_xirr + elif contribution > ZERO: + basis, rate = BASIS_CONTRIBUTION, ZERO + else: + basis, rate = BASIS_NONE, None + + projection = project( + current=current, + target=target, + as_of=as_of, + annual_rate=rate, + monthly_contribution=contribution, + basis=basis, + ) + progress = (current / target).quantize(RATE_PLACES) if target > ZERO else ZERO + progress = max(ZERO, progress) + + needed: Decimal | None = None + on_track: bool | None = None + if target_date is not None: + needed = monthly_needed( + current=current, + target=target, + months=months_between(as_of, target_date), + annual_rate=rate, + ) + projected = projection.projected_date + on_track = projected is not None and projected <= target_date + + return GoalProgress( + goal_id=goal_id, + as_of=as_of, + current_value_rub=current, + target_amount_rub=target, + progress=progress, + projected_date=projection.projected_date, + basis=projection.basis, + assumed_rate=( + projection.assumed_rate.quantize(RATE_PLACES) + if projection.assumed_rate is not None + else None + ), + monthly_needed_rub=needed, + on_track=on_track, + ) + + +# --------------------------------------------------------------------------- loading + + +async def _scope_values(session: AsyncSession) -> dict[str, Decimal]: + """Latest total per scope, from the daily value series.""" + rows = ( + await session.execute( + select( + MetricPortfolioValueDaily.scope, + MetricPortfolioValueDaily.d, + MetricPortfolioValueDaily.total_rub, + ).order_by(MetricPortfolioValueDaily.scope, MetricPortfolioValueDaily.d) + ) + ).all() + out: dict[str, Decimal] = {} + for r in rows: + out[r.scope] = Decimal(r.total_rub) + return out + + +async def _scope_rates(session: AsyncSession) -> dict[str, Decimal]: + """Trailing XIRR per scope, chosen by `pick_rate`.""" + rows = ( + await session.execute( + select( + MetricReturns.scope, + MetricReturns.period, + MetricReturns.date_from, + MetricReturns.date_to, + MetricReturns.xirr, + ) + ) + ).all() + per_scope: dict[str, list[dict[str, object]]] = {} + for r in rows: + per_scope.setdefault(r.scope, []).append( + { + "period": r.period, + "date_from": r.date_from, + "date_to": r.date_to, + "xirr": r.xirr, + } + ) + out: dict[str, Decimal] = {} + for scope, scope_rows in per_scope.items(): + rate, _ = pick_rate(scope_rows) + if rate is not None: + out[scope] = rate + return out + + +async def compute_goal_progress(session: AsyncSession, goal: Goal) -> GoalProgress | None: + """Progress of one goal; None when its target cannot be expressed in RUB.""" + as_of = today_local() + values = await _scope_values(session) + rates = await _scope_rates(session) + return await _evaluate_goal(session, goal, as_of, values, rates) + + +async def _evaluate_goal( + session: AsyncSession, + goal: Goal, + as_of: date, + values: Mapping[str, Decimal], + rates: Mapping[str, Decimal], +) -> GoalProgress | None: + target = Decimal(goal.target_amount) + if (goal.currency or "RUB").upper() != "RUB": + fx = await FxTable.load(session) + converted = fx.to_rub(target, goal.currency, as_of) + if converted is None: + FINDINGS.add( + "goal_without_fx", + "warn", + f"Цель «{goal.name}» задана в {goal.currency}, но курса на {as_of} нет — " + f"прогресс не посчитан", + ref={"goal_id": goal.id}, + ) + return None + target = converted + if target <= ZERO: + FINDINGS.add( + "goal_without_target", + "warn", + f"У цели «{goal.name}» неположительная сумма — прогресс не посчитан", + ref={"goal_id": goal.id}, + ) + return None + + current = values.get(goal.scope) + if current is None: + FINDINGS.add( + "goal_scope_without_metrics", + "warn", + f"Для цели «{goal.name}» нет метрик по scope {goal.scope} — прогресс нулевой", + ref={"goal_id": goal.id, "scope": goal.scope}, + ) + current = ZERO + + return evaluate( + goal_id=goal.id, + as_of=as_of, + current=current, + target=target, + target_date=goal.target_date, + monthly_contribution=( + Decimal(goal.monthly_contribution) if goal.monthly_contribution is not None else None + ), + trailing_xirr=rates.get(goal.scope), + ) + + +# --------------------------------------------------------------------------- refresh step + + +async def rebuild_goal_progress(session: AsyncSession) -> None: + """Replace `metric_goal_progress` for every goal that is not archived.""" + await session.execute(delete(MetricGoalProgress)) + + goals = ( + (await session.execute(select(Goal).where(Goal.archived.is_(False)).order_by(Goal.id))) + .scalars() + .all() + ) + if not goals: + return + + as_of = today_local() + values = await _scope_values(session) + rates = await _scope_rates(session) + + rows: list[dict[str, object]] = [] + for goal in goals: + progress = await _evaluate_goal(session, goal, as_of, values, rates) + if progress is None: + continue + rows.append( + { + "goal_id": progress.goal_id, + "as_of": progress.as_of, + "current_value_rub": progress.current_value_rub, + "target_amount_rub": progress.target_amount_rub, + "progress": progress.progress, + "projected_date": progress.projected_date, + "basis": progress.basis, + "assumed_rate": progress.assumed_rate, + "monthly_needed_rub": progress.monthly_needed_rub, + "on_track": progress.on_track, + } + ) + if rows: + await session.execute(insert(MetricGoalProgress), rows) + log.info("goals: %s rows", len(rows)) diff --git a/backend/src/fintracker/analytics/income.py b/backend/src/fintracker/analytics/income.py new file mode 100644 index 0000000..3960eb8 --- /dev/null +++ b/backend/src/fintracker/analytics/income.py @@ -0,0 +1,947 @@ +"""Dividends and coupons: what already landed, and what is expected (plan §3). + +Two tables come out of here and they answer different questions: + +* `metric_income_monthly` — money actually received, per month, kind and currency. Pure + history, read from `event` (confirmed only, like every metric), never from a forecast. +* `metric_income_calendar` — one row per payment, past and future. The past rows carry + `basis = "paid"`; the future ones carry the assumption that produced them. + +**`basis` is the point of this module.** A bond's coupon is arithmetic on a published +schedule; an announced dividend is a fact the issuer stated; everything else is an +extrapolation from the last 24 months that can be wrong by any amount. They are computed by +three different code paths on purpose, and the screen shows which one produced each number — +a single «ожидаемый доход» that mixes them is a guess wearing a forecast's clothes. + +What is deliberately NOT here: + +* **No zero rows.** An instrument nobody ever paid on, and that has no schedule, does not get + a row with `amount = 0`; it gets a `FINDINGS` warning and stays out of the forecast. A zero + looks like knowledge — «this pays nothing» — which is the opposite of what we have. +* **No substituted FX.** A payment whose date has no rate keeps its native amount and gets + `amount_rub = NULL` plus a finding, exactly like the rest of the analytics. Future dates + convert at today's rate (nobody quotes tomorrow's), which is stated rather than hidden. +* **No position we do not hold.** The forecast multiplies by `lot.qty_remaining`, so a paper + sold in full disappears from it and a half-sold one halves. + +Priority between corporate-action sources (T-Invest vs MOEX) is NOT decided here. It belongs +to `fintracker.pricing.payouts.resolve_payouts`, which every declared payout is passed +through before it is read. The import is guarded because the two modules were written in +parallel: when the resolver is absent we fall back to a local `paid > announced > forecast` +preference with no cross-source rule at all. **That fallback is a stopgap, not a second +opinion** — when `pricing/payouts.py` is importable it decides, and nothing here may grow a +competing rule. +""" + +from __future__ import annotations + +import calendar as _calendar +import logging +from collections import defaultdict +from collections.abc import Iterable, Mapping, Sequence +from dataclasses import dataclass, replace +from datetime import date, timedelta +from decimal import Decimal +from itertools import pairwise +from typing import Any + +from sqlalchemy import delete, insert, select +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import FINDINGS, today_local +from fintracker.analytics.valuation import account_scopes +from fintracker.models import ( + AssetClass, + BondNominalSchedule, + CorporateAction, + CorporateActionStatus, + Event, + EventStatus, + IncomeBasis, + Instrument, + Lot, + MetricIncomeCalendar, + MetricIncomeMonthly, +) +from fintracker.models.ledger import POSITION_KINDS, EventKind +from fintracker.pricing.fx import FxTable + +try: # pragma: no cover - exercised by whichever half of the tree is on disk + from fintracker.pricing.payouts import resolve_payouts # type: ignore[attr-defined] +except ImportError: # the corporate-action sources are written in parallel; see module docstring + resolve_payouts = None # type: ignore[assignment] + +log = logging.getLogger(__name__) + +ZERO = Decimal(0) +RUB = "RUB" + +#: How far ahead the calendar is built. The API caps `/income/forecast` at 36 months, so the +#: table has to reach that far or the endpoint would answer with a silently truncated series. +FORECAST_MONTHS = 36 + +#: How far back periodicity is inferred from. Two years is the plan's window: one is not +#: enough to see an annual payer twice, three drags in a dividend policy that no longer holds. +HISTORY_MONTHS = 24 + +#: Payments per year the history path is allowed to conclude (plan §3). A monthly payer is +#: therefore rounded to quarterly — bonds avoid that by going through the schedule path, and +#: `income_irregular_history` says so when the observed spacing does not fit. +FREQUENCIES = (1, 2, 4) + +#: Event kinds that are income. `repayment` is one of them: the redemption of a bond pays +#: money, and the calendar is about money landing, not about the position surviving. +INCOME_KINDS: tuple[EventKind, ...] = ( + EventKind.dividend, + EventKind.coupon, + EventKind.amortization, + EventKind.repayment, +) + +#: Instruments that can pay at all — the ones a missing schedule is worth warning about. +PAYING_CLASSES = frozenset({AssetClass.share, AssetClass.bond, AssetClass.etf, AssetClass.fund}) + + +# -------------------------------------------------------------------------------------- +# pure core: plain values in, calendar entries out — no session, no I/O +# -------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class Payment: + """One payout that actually happened, as the ledger recorded it.""" + + account_id: int + instrument_id: int + kind: str + d: date + currency: str + amount: Decimal + """Net cash that reached the account; `tax` is already taken out of it.""" + tax: Decimal + held_qty: Decimal + """Position the account held on `d` — what turns a payment into an amount per unit.""" + + +@dataclass(frozen=True) +class Payout: + """A declared or projected payout — one `corporate_action`, flattened. + + The field set is `pricing.payouts.PayoutLike`, so a resolved row and a hand-built one are + interchangeable and the resolver can be handed either. + """ + + instrument_id: int + kind: str + status: str + record_date: date | None + pay_date: date | None + amount_per_unit: Decimal | None + currency: str | None + ex_date: date | None = None + source: str = "" + + @property + def effective_date(self) -> date | None: + """When the money is expected: pay date, else ex-date, else the record date.""" + return self.pay_date or self.ex_date or self.record_date + + +@dataclass(frozen=True) +class Entry: + """One calendar row before it knows which scope it belongs to.""" + + instrument_id: int + kind: str + expected_date: date + record_date: date | None + qty: Decimal + per_unit: Decimal | None + amount: Decimal + currency: str + basis: IncomeBasis + tax_withheld: Decimal | None = None + + +def add_months(d: date, months: int) -> date: + """`d` shifted by whole months, clamped to the length of the target month.""" + total = d.year * 12 + (d.month - 1) + months + year, month = divmod(total, 12) + month += 1 + return date(year, month, min(d.day, _calendar.monthrange(year, month)[1])) + + +def month_start(d: date) -> date: + return d.replace(day=1) + + +def nominal_at(schedule: Sequence[tuple[date, Decimal]], d: date) -> Decimal | None: + """The nominal in force on `d`: the last entry that had already taken effect. + + `bond_nominal_schedule` is ordered by `effective_date` and each row says what the nominal + *becomes*, so a date before the first row has no answer here — the caller falls back to + `instrument.nominal`. + """ + current: Decimal | None = None + for effective, value in schedule: + if effective > d: + break + current = value + return current + + +def coupon_per_unit( + declared: Decimal, base_nominal: Decimal | None, nominal_on_date: Decimal | None +) -> Decimal: + """A published coupon rescaled to the nominal that stands on the coupon's own date. + + A coupon is a rate on the nominal, and an amortisation cuts the nominal — so a schedule + published against the issue par overstates every coupon after the first amortisation, by + exactly the fraction already repaid. This is the single most common way to get a bond + forecast wrong, which is why it is one function with one test. + + Scaling is skipped when either nominal is unknown or when they are equal (the ordinary + non-amortised bond), so a source that already publishes amortised coupons is not cut + twice as long as it also publishes the matching nominal schedule. + """ + if base_nominal is None or nominal_on_date is None or base_nominal <= ZERO: + return declared + if nominal_on_date == base_nominal: + return declared + return declared * nominal_on_date / base_nominal + + +def detect_frequency(dates: Sequence[date]) -> int | None: + """Payments per year implied by `dates`, snapped to 1, 2 or 4 — or None if unknowable. + + One payment says nothing about spacing, so it is read as annual: that is the commonest + Russian dividend and the alternative (dropping the paper) hides a payout we have actually + seen. Everything here is labelled `history` downstream, which is the honest part. + + Where it breaks: a payer that changed policy inside the window (two payments a year, then + one) averages into something it never was, and a monthly payer is rounded down to + quarterly because the plan admits only three buckets. `regular()` is what flags the first + case; the second only reaches here for a bond with no schedule at all. + """ + unique = sorted(set(dates)) + if not unique: + return None + if len(unique) == 1: + return 1 + span = (unique[-1] - unique[0]).days + if span <= 0: + return None + interval = Decimal(span) / Decimal(len(unique) - 1) + per_year = Decimal(365) / interval + return min(FREQUENCIES, key=lambda f: abs(per_year - Decimal(f))) + + +def regular(dates: Sequence[date]) -> bool: + """True when the gaps between payments are within 2x of each other. + + An irregular history still produces a forecast — refusing to guess at all would empty the + screen for every special dividend — but it is reported, so the number carries its doubt. + """ + unique = sorted(set(dates)) + if len(unique) < 3: + return True + gaps = [(b - a).days for a, b in pairwise(unique)] + smallest, largest = min(gaps), max(gaps) + return smallest > 0 and Decimal(largest) <= Decimal(smallest) * 2 + + +def project_dates(last: date, frequency: int, *, start: date, end: date) -> list[date]: + """Future payment dates implied by a frequency, inside [start, end].""" + step = 12 // frequency + out: list[date] = [] + k = 1 + while True: + d = add_months(last, step * k) + if d > end: + break + if d >= start: + out.append(d) + k += 1 + return out + + +def fold_payments(payments: Iterable[Payment]) -> list[Entry]: + """Received payouts as `paid` calendar entries, one per (instrument, kind, date, currency). + + Folding across accounts is what makes a scope's calendar readable: the same dividend paid + into a brokerage and an ИИС account is one event in the world, and two rows would read as + two dividends. + """ + grouped: dict[tuple[int, str, date, str], list[Payment]] = defaultdict(list) + for p in payments: + grouped[(p.instrument_id, p.kind, p.d, p.currency)].append(p) + + out: list[Entry] = [] + for (instrument_id, kind, d, currency), group in sorted(grouped.items()): + amount = sum((p.amount for p in group), start=ZERO) + tax = sum((p.tax for p in group), start=ZERO) + qty = sum((p.held_qty for p in group), start=ZERO) + out.append( + Entry( + instrument_id=instrument_id, + kind=kind, + expected_date=d, + record_date=None, + qty=qty, + per_unit=amount / qty if qty > ZERO else None, + amount=amount, + currency=currency, + basis=IncomeBasis.paid, + tax_withheld=tax, + ) + ) + return out + + +MonthlyRow = tuple[Decimal, Decimal, int] +"""(amount, tax withheld, payment count) for one (month, kind, currency).""" + + +def monthly_rows(payments: Iterable[Payment]) -> dict[tuple[date, str, str], MonthlyRow]: + """(month, kind, currency) -> (amount, tax withheld, payment count).""" + out: dict[tuple[date, str, str], MonthlyRow] = {} + for p in payments: + key = (month_start(p.d), p.kind, p.currency) + amount, tax, count = out.get(key, (ZERO, ZERO, 0)) + out[key] = (amount + p.amount, tax + p.tax, count + 1) + return out + + +@dataclass(frozen=True) +class BondFacts: + """Everything the schedule path needs about one bond.""" + + nominal: Decimal | None + """Issue par, the base a published coupon is quoted against.""" + nominal_schedule: tuple[tuple[date, Decimal], ...] + maturity_date: date | None + currency: str + + +def bond_entries( + instrument_id: int, + facts: BondFacts, + coupons: Sequence[Payout], + qty: Decimal, + *, + start: date, + end: date, +) -> list[Entry]: + """Coupons, amortisations and the redemption of one bond — the `schedule` basis. + + All three come from published arithmetic rather than from history: the coupon from the + payment schedule scaled by `coupon_per_unit`, the amortisation from the step in + `bond_nominal_schedule` itself (the size of the step *is* the payment), the redemption + from the maturity date and the nominal that survives to it. + """ + schedule = list(facts.nominal_schedule) + base = schedule[0][1] if schedule else facts.nominal + out: list[Entry] = [] + + for payout in coupons: + d = payout.effective_date + if d is None or not (start <= d <= end) or payout.amount_per_unit is None: + continue + per_unit = coupon_per_unit( + Decimal(payout.amount_per_unit), base, nominal_at(schedule, d) or facts.nominal + ) + out.append( + Entry( + instrument_id=instrument_id, + kind="coupon", + expected_date=d, + record_date=payout.record_date, + qty=qty, + per_unit=per_unit, + amount=per_unit * qty, + currency=(payout.currency or facts.currency).upper(), + basis=IncomeBasis.schedule, + ) + ) + + previous: Decimal | None = None + for effective, value in schedule: + if previous is not None and start <= effective <= end and value < previous: + step = previous - value + out.append( + Entry( + instrument_id=instrument_id, + kind="amortization", + expected_date=effective, + record_date=None, + qty=qty, + per_unit=step, + amount=step * qty, + currency=facts.currency.upper(), + basis=IncomeBasis.schedule, + ) + ) + previous = value + + maturity = facts.maturity_date + if maturity is not None and start <= maturity <= end: + par = nominal_at(schedule, maturity) or facts.nominal + if par is not None: + out.append( + Entry( + instrument_id=instrument_id, + kind="repayment", + expected_date=maturity, + record_date=None, + qty=qty, + per_unit=par, + amount=par * qty, + currency=facts.currency.upper(), + basis=IncomeBasis.schedule, + ) + ) + return out + + +def announced_entries( + instrument_id: int, payouts: Sequence[Payout], qty: Decimal, *, start: date, end: date +) -> list[Entry]: + """Declared payouts with a future date — a fact from the issuer, not an extrapolation.""" + out: list[Entry] = [] + for payout in payouts: + if payout.status != CorporateActionStatus.announced.value: + continue + d = payout.effective_date + if d is None or not (start <= d <= end) or payout.amount_per_unit is None: + continue + per_unit = Decimal(payout.amount_per_unit) + out.append( + Entry( + instrument_id=instrument_id, + kind=payout.kind, + expected_date=d, + record_date=payout.record_date, + qty=qty, + per_unit=per_unit, + amount=per_unit * qty, + currency=(payout.currency or RUB).upper(), + basis=IncomeBasis.announced, + ) + ) + return out + + +#: An extrapolated date this close to an announced one is the same payment, and loses. +SAME_PAYMENT_DAYS = 45 + + +def history_entries( + instrument_id: int, + payments: Sequence[Payment], + qty: Decimal, + *, + start: date, + end: date, +) -> tuple[list[Entry], bool]: + """Extrapolation from the last payments: the honest guess, labelled `history`. + + Returns the entries and whether the history looked regular. The amount is the LAST + payment's per-unit amount times the position held now — not the last total, which would + keep paying on shares already sold. + """ + per_kind: dict[str, list[Payment]] = defaultdict(list) + for p in payments: + per_kind[p.kind].append(p) + + out: list[Entry] = [] + steady = True + for kind, group in sorted(per_kind.items()): + by_date: dict[date, tuple[Decimal, Decimal, str]] = {} + for p in group: + amount, held, _ = by_date.get(p.d, (ZERO, ZERO, p.currency)) + by_date[p.d] = (amount + p.amount, held + p.held_qty, p.currency) + dates = sorted(by_date) + frequency = detect_frequency(dates) + if frequency is None: + continue + steady = steady and regular(dates) + last = dates[-1] + amount, held, currency = by_date[last] + per_unit = amount / held if held > ZERO else None + if per_unit is None: + continue + for d in project_dates(last, frequency, start=start, end=end): + out.append( + Entry( + instrument_id=instrument_id, + kind=kind, + expected_date=d, + record_date=None, + qty=qty, + per_unit=per_unit, + amount=per_unit * qty, + currency=currency.upper(), + basis=IncomeBasis.history, + ) + ) + return out, steady + + +def drop_shadowed(announced: Sequence[Entry], projected: Sequence[Entry]) -> list[Entry]: + """Extrapolated entries that an announced payout already covers are removed. + + An issuer that has declared its autumn dividend has answered the question history was + guessing at, so the guess goes — but the *other* payment of a twice-a-year payer, which + nobody has declared yet, stays. Dropping every projection the moment one date is + announced would quietly halve the year. + """ + if not announced: + return list(projected) + out: list[Entry] = [] + for entry in projected: + clash = any( + a.kind == entry.kind + and abs((a.expected_date - entry.expected_date).days) <= SAME_PAYMENT_DAYS + for a in announced + ) + if not clash: + out.append(entry) + return out + + +def _payout_day(payout: object) -> date | None: + for attr in ("pay_date", "ex_date", "record_date"): + value = getattr(payout, attr, None) + if value is not None: + return value + return None + + +def resolve_actions_fallback(payouts: Sequence[Any]) -> list[Any]: + """Stopgap for `pricing.payouts.resolve_payouts` (see the module docstring). + + One payment can be described by both T-Invest and MOEX; without a rule the calendar shows + it twice. With no resolver on disk we keep the best-known status per (instrument, kind, + date) — `paid` over `announced` over `forecast` — and drop `cancelled` outright. + Cross-source priority is explicitly NOT decided here, and this path exists only so that + the income step still runs when `pricing/payouts.py` is missing. + """ + rank = { + CorporateActionStatus.paid: 3, + CorporateActionStatus.announced: 2, + CorporateActionStatus.forecast: 1, + } + best: dict[tuple[int, str, date | None], Any] = {} + for payout in payouts: + if payout.status == CorporateActionStatus.cancelled: + continue + key = (payout.instrument_id, str(payout.kind), _payout_day(payout)) + current = best.get(key) + if current is None or rank.get(payout.status, 0) > rank.get(current.status, 0): + best[key] = payout + return sorted( + best.values(), + key=lambda p: (p.instrument_id, str(p.kind), _payout_day(p) or date.min), + ) + + +def resolve_actions(payouts: Sequence[Any]) -> list[Any]: + """Delegate to `pricing.payouts.resolve_payouts` when it exists, else fall back.""" + if resolve_payouts is None: + return resolve_actions_fallback(payouts) + try: + return list(resolve_payouts(payouts)) + except (AttributeError, TypeError, ValueError): # not the shape we guessed at + log.warning("resolve_payouts rejected our call shape; using the local fallback") + return resolve_actions_fallback(payouts) + + +# -------------------------------------------------------------------------------------- +# the refresh step +# -------------------------------------------------------------------------------------- + + +async def rebuild_income(session: AsyncSession) -> None: + """Replace `metric_income_calendar` and `metric_income_monthly` for every scope.""" + await session.execute(delete(MetricIncomeCalendar)) + await session.execute(delete(MetricIncomeMonthly)) + + payments = await _load_payments(session) + open_qty = await _open_quantities(session) + accounts = {p.account_id for p in payments} | {a for a, _ in open_qty} + accounts |= await _ledger_accounts(session) + scopes = await account_scopes(session, accounts) + if not scopes: + return + + instruments = await _load_instruments(session) + schedules = await _load_nominal_schedules(session) + payouts = await _load_payouts(session) + fx = await FxTable.load(session) + + today = today_local() + horizon = add_months(today, FORECAST_MONTHS) + since = add_months(today, -HISTORY_MONTHS) + + calendar_rows: list[dict[str, object]] = [] + monthly_out: list[dict[str, object]] = [] + missing_fx: set[int] = set() + + for scope, account_ids in sorted(scopes.items()): + scoped = [p for p in payments if p.account_id in account_ids] + qty_by_instrument: dict[int, Decimal] = defaultdict(lambda: ZERO) + for (account_id, instrument_id), qty in open_qty.items(): + if account_id in account_ids: + qty_by_instrument[instrument_id] += qty + + entries = fold_payments(scoped) + entries += _forecast( + scope=scope, + payments=[p for p in scoped if p.d >= since], + qty_by_instrument=qty_by_instrument, + instruments=instruments, + schedules=schedules, + payouts=payouts, + today=today, + horizon=horizon, + ) + for entry in _dedupe(entries): + rub = fx.to_rub(entry.amount, entry.currency, min(entry.expected_date, today)) + if rub is None: + missing_fx.add(entry.instrument_id) + calendar_rows.append( + { + "scope": scope, + "instrument_id": entry.instrument_id, + "kind": entry.kind, + "expected_date": entry.expected_date, + "record_date": entry.record_date, + "qty": entry.qty, + "per_unit": entry.per_unit, + "amount": entry.amount, + "currency": entry.currency, + "amount_rub": rub, + "basis": entry.basis, + "tax_withheld": entry.tax_withheld, + } + ) + + for (month, kind, currency), (amount, tax, count) in sorted(monthly_rows(scoped).items()): + rub = _month_rub(fx, scoped, month, kind, currency) + monthly_out.append( + { + "scope": scope, + "month": month, + "kind": kind, + "currency": currency, + "amount": amount, + "amount_rub": rub, + "tax_withheld": tax, + "payment_count": count, + } + ) + + if calendar_rows: + await session.execute(insert(MetricIncomeCalendar), calendar_rows) + if monthly_out: + await session.execute(insert(MetricIncomeMonthly), monthly_out) + if missing_fx: + FINDINGS.add( + "income_missing_fx", + "warn", + f"У {len(missing_fx)} инструментов выплата в валюте без курса на дату — " + f"строка календаря есть, рублёвая сумма пустая", + count=len(missing_fx), + ref={"instruments": sorted(missing_fx)}, + ) + log.info("income: %s calendar rows, %s monthly rows", len(calendar_rows), len(monthly_out)) + + +def _month_rub( + fx: FxTable, payments: Sequence[Payment], month: date, kind: str, currency: str +) -> Decimal | None: + """A month's total in RUB — NULL as soon as one payment in it had no rate.""" + total = ZERO + for p in payments: + if month_start(p.d) != month or p.kind != kind or p.currency != currency: + continue + rub = fx.to_rub(p.amount, p.currency, p.d) + if rub is None: + return None + total += rub + return total + + +def _forecast( + *, + scope: str, + payments: Sequence[Payment], + qty_by_instrument: Mapping[int, Decimal], + instruments: Mapping[int, Instrument], + schedules: Mapping[int, tuple[tuple[date, Decimal], ...]], + payouts: Mapping[int, list[Payout]], + today: date, + horizon: date, +) -> list[Entry]: + """The forward half of the calendar for one scope.""" + by_instrument: dict[int, list[Payment]] = defaultdict(list) + for p in payments: + by_instrument[p.instrument_id].append(p) + + out: list[Entry] = [] + silent: list[int] = [] + irregular: list[int] = [] + + for instrument_id, qty in sorted(qty_by_instrument.items()): + instrument = instruments.get(instrument_id) + if instrument is None or qty <= ZERO: + continue + + instrument_payouts = payouts.get(instrument_id, []) + entries: list[Entry] = [] + if instrument.asset_class == AssetClass.bond: + facts = BondFacts( + nominal=Decimal(instrument.nominal) if instrument.nominal is not None else None, + nominal_schedule=schedules.get(instrument_id, ()), + maturity_date=instrument.maturity_date, + currency=(instrument.nominal_currency or instrument.currency or RUB), + ) + coupons = [ + p + for p in instrument_payouts + if p.kind == "coupon" and p.status != CorporateActionStatus.cancelled + ] + entries = bond_entries(instrument_id, facts, coupons, qty, start=today, end=horizon) + + if not entries: + announced = announced_entries( + instrument_id, instrument_payouts, qty, start=today, end=horizon + ) + projected, steady = history_entries( + instrument_id, + by_instrument.get(instrument_id, []), + qty, + start=today, + end=horizon, + ) + if not steady: + irregular.append(instrument_id) + entries = announced + drop_shadowed(announced, projected) + + if not entries: + if instrument.asset_class in PAYING_CLASSES: + silent.append(instrument_id) + continue + out.extend(entries) + + # findings are about the portfolio, not about each slice of it — only `all` reports + if scope == "all": + if silent: + FINDINGS.add( + "income_without_history", + "warn", + f"У {len(silent)} инструментов нет ни графика выплат, ни истории — " + f"в прогноз доходов не вошли", + count=len(silent), + ref={"instruments": sorted(silent)}, + ) + if irregular: + FINDINGS.add( + "income_irregular_history", + "info", + f"У {len(irregular)} инструментов выплаты нерегулярны — " + f"периодичность в прогнозе определена приблизительно", + count=len(irregular), + ref={"instruments": sorted(irregular)}, + ) + return out + + +def _dedupe(entries: Sequence[Entry]) -> list[Entry]: + """Collapse entries onto the table's key: (instrument, kind, date, basis). + + Two sources describing one payment, or two currencies on one day, would otherwise violate + the unique constraint. Same-currency duplicates are summed (two accounts, one dividend); + a genuine currency clash keeps the larger and is rare enough to leave unreported. + """ + merged: dict[tuple[int, str, date, IncomeBasis], Entry] = {} + for entry in entries: + key = (entry.instrument_id, entry.kind, entry.expected_date, entry.basis) + current = merged.get(key) + if current is None: + merged[key] = entry + continue + if current.currency != entry.currency: + if abs(entry.amount) > abs(current.amount): + merged[key] = entry + continue + tax = None + if current.tax_withheld is not None or entry.tax_withheld is not None: + tax = (current.tax_withheld or ZERO) + (entry.tax_withheld or ZERO) + merged[key] = replace( + current, + qty=current.qty + entry.qty, + amount=current.amount + entry.amount, + tax_withheld=tax, + ) + return sorted(merged.values(), key=lambda e: (e.expected_date, e.instrument_id, e.kind)) + + +# -------------------------------------------------------------------------------------- +# loaders +# -------------------------------------------------------------------------------------- + + +async def _ledger_accounts(session: AsyncSession) -> set[int]: + return set( + ( + await session.execute( + select(Event.account_id).where(Event.status == EventStatus.confirmed).distinct() + ) + ) + .scalars() + .all() + ) + + +#: How far before a payment to look for the position, when the pay date already shows none. +#: A dividend is earned on the record date and can arrive weeks after the shares were sold; +#: reading only the pay date would make that payment look like it came from nowhere. +EX_DATE_LOOKBACK_DAYS = 45 + + +PositionSeries = dict[tuple[int, int], list[tuple[date, Decimal]]] + + +async def _position_series(session: AsyncSession) -> PositionSeries: + """(account, instrument) -> ordered (date, signed quantity change) from the ledger.""" + rows = ( + await session.execute( + select(Event.account_id, Event.instrument_id, Event.trade_date, Event.quantity) + .where( + Event.status == EventStatus.confirmed, + Event.kind.in_(tuple(POSITION_KINDS)), + Event.instrument_id.is_not(None), + Event.quantity.is_not(None), + ) + .order_by(Event.trade_date) + ) + ).all() + out: dict[tuple[int, int], list[tuple[date, Decimal]]] = defaultdict(list) + for r in rows: + out[(r.account_id, r.instrument_id)].append((r.trade_date, Decimal(r.quantity))) + return out + + +def held_at(series: Sequence[tuple[date, Decimal]], d: date) -> Decimal: + """Position on `d`: every quantity change up to and including that day, added up.""" + total = ZERO + for day, delta in series: + if day > d: + break + total += delta + return total + + +async def _load_payments(session: AsyncSession) -> list[Payment]: + """Every confirmed payout event — the only source the history half ever reads.""" + positions = await _position_series(session) + rows = ( + await session.execute( + select( + Event.account_id, + Event.instrument_id, + Event.kind, + Event.trade_date, + Event.currency, + Event.amount, + Event.tax, + Event.quantity, + ).where( + Event.status == EventStatus.confirmed, + Event.kind.in_(INCOME_KINDS), + Event.instrument_id.is_not(None), + ) + ) + ).all() + out: list[Payment] = [] + for r in rows: + series = positions.get((r.account_id, r.instrument_id), []) + held = held_at(series, r.trade_date) + if held <= ZERO: + held = held_at(series, r.trade_date - timedelta(days=EX_DATE_LOOKBACK_DAYS)) + out.append( + Payment( + account_id=r.account_id, + instrument_id=r.instrument_id, + kind=str(r.kind.value if hasattr(r.kind, "value") else r.kind), + d=r.trade_date, + currency=(r.currency or RUB).upper(), + amount=Decimal(r.amount or 0), + tax=Decimal(r.tax or 0), + held_qty=max(held, ZERO), + ) + ) + return out + + +async def _open_quantities(session: AsyncSession) -> dict[tuple[int, int], Decimal]: + """Position per (account, instrument) from the open lots — the forecast's multiplier. + + Lots, not the event replay: they are split- and amortisation-aware, and a paper that is + fully sold simply has no open lot, which is exactly the «not in the forecast» rule. + """ + rows = ( + await session.execute( + select(Lot.account_id, Lot.instrument_id, Lot.qty_remaining).where( + Lot.qty_remaining != 0 + ) + ) + ).all() + out: dict[tuple[int, int], Decimal] = defaultdict(lambda: ZERO) + for r in rows: + out[(r.account_id, r.instrument_id)] += Decimal(r.qty_remaining) + return {k: v for k, v in out.items() if v != ZERO} + + +async def _load_instruments(session: AsyncSession) -> dict[int, Instrument]: + rows = (await session.execute(select(Instrument))).scalars().all() + return {r.id: r for r in rows} + + +async def _load_nominal_schedules( + session: AsyncSession, +) -> dict[int, tuple[tuple[date, Decimal], ...]]: + rows = ( + await session.execute( + select( + BondNominalSchedule.instrument_id, + BondNominalSchedule.effective_date, + BondNominalSchedule.nominal, + ).order_by(BondNominalSchedule.instrument_id, BondNominalSchedule.effective_date) + ) + ).all() + out: dict[int, list[tuple[date, Decimal]]] = defaultdict(list) + for r in rows: + out[r.instrument_id].append((r.effective_date, Decimal(r.nominal))) + return {k: tuple(v) for k, v in out.items()} + + +async def _load_payouts(session: AsyncSession) -> dict[int, list[Payout]]: + """Declared payouts per instrument, after source resolution (see the module docstring).""" + rows = (await session.execute(select(CorporateAction))).scalars().all() + out: dict[int, list[Payout]] = defaultdict(list) + for r in resolve_actions(list(rows)): + payout = Payout( + instrument_id=r.instrument_id, + kind=str(r.kind.value if hasattr(r.kind, "value") else r.kind), + status=str(r.status.value if hasattr(r.status, "value") else r.status), + record_date=r.record_date, + pay_date=r.pay_date, + amount_per_unit=Decimal(r.amount_per_unit) if r.amount_per_unit is not None else None, + currency=r.currency, + ex_date=getattr(r, "ex_date", None), + source=getattr(r, "source", "") or "", + ) + out[payout.instrument_id].append(payout) + return out diff --git a/backend/src/fintracker/analytics/rebalance.py b/backend/src/fintracker/analytics/rebalance.py new file mode 100644 index 0000000..d13b585 --- /dev/null +++ b/backend/src/fintracker/analytics/rebalance.py @@ -0,0 +1,603 @@ +"""Target weights versus reality, and the trades that close the gap (plan §3, фаза 4). + +The input is a `portfolio_target` set — the user's decision — and `metric_allocation` — the +measurement `analytics/allocation.py` has just written. Nothing here re-derives a weight: +the four allocation dimensions already cover the same total (securities plus cash), so a +bucket's share of the portfolio means the same thing on this screen as on the pie chart. + +**Which paper is traded inside a bucket.** A target is set on a bucket, never on a paper. +«Акции — 60 %» says nothing about whether the 60 % should be Сбер or Газпром, so the +rebalancer trims and tops up **proportionally to what the bucket already holds**: every +position keeps its share of the bucket, and only the bucket's size moves. The alternatives +were considered and rejected: + +* *sell the most overweight, buy the most underweight* — overweight relative to what? There + is no per-instrument target, so the rule silently invents one (equal weight inside the + bucket) and quietly reshapes the portfolio while claiming to only resize it; +* *sell the biggest position first* — fewer orders, but it concentrates the whole tax event + on one lot and drifts the bucket's composition with every rebalance. + +Proportional is the only rule that uses nothing the user did not say. Its cost is more +orders; whole-lot rounding cuts most of them away by itself, and a bucket whose share is too +small to buy one lot simply gets no trade. + +**Rounding always goes toward doing less.** Quantities are floored to whole lots, so acting +on every suggestion can neither overshoot the target nor spend money that is not there. An +instrument with `lot = 10` is never suggested in sevens: a recommendation that cannot be +sent to the broker is not a recommendation. + +**Sale proceeds do not fund purchases.** The buy side is capped by the cash that exists +right now (`cash_available_rub`, overridable for what-if), not by cash plus whatever the +sells would bring in. The trades are a proposal, not a sequenced plan, and money from a sale +that has not happened is exactly the unexecutable recommendation this module exists to +avoid. Where the cash runs out the quantity is cut to what it covers and `blocked_by_cash` +is set, rather than the trade being dropped — the user should see what the cash is blocking. + +**Shorts are never proposed.** A sell is capped by the position actually held (itself +floored to whole lots), so the worst case is a position closed to zero. + +**Tax is not modelled here.** The suggested sells ignore what they would cost in tax; by +default we optimise the portfolio's shape, not its tax bill. That is a real omission and it +has a direction: a lot younger than three years loses the ЛДВ exemption of art. 219.1 НК, so +selling it is strictly more expensive than selling an older lot of the same paper, and a +proportional trim will happily pick the young one. `analytics/tax.py` owns the per-lot +picture (`ldv_date`, `tax_if_sold_now_rub`); until a rebalance consults it, treat a sell +suggestion as «this much of this bucket», not «this exact lot». + +**A bucket inside its band produces nothing.** `portfolio_target.band` is the tolerance +around the target; within it `within_band` is true and `suggested_qty` is NULL. Without a +band every tick of every price would produce a trade recommendation. + +An instrument with no usable price is left out of the trades entirely and reported through +`FINDINGS` — never a row with a zero in it, which would read as «sell nothing» rather than +«we do not know». +""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from collections.abc import Mapping, Sequence +from dataclasses import dataclass, field +from datetime import date +from decimal import ROUND_FLOOR, Decimal + +from sqlalchemy import delete, insert, select, update +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import FINDINGS, today_local +from fintracker.analytics.allocation import CASH, Holding, bucket_of +from fintracker.analytics.valuation import cash_balances +from fintracker.models import ( + AllocationDimension, + Instrument, + MetricAllocation, + MetricHolding, + MetricRebalance, + PortfolioAccount, + PortfolioTarget, +) +from fintracker.pricing.fx import FxTable + +log = logging.getLogger(__name__) + +ZERO = Decimal(0) +ONE = Decimal(1) + +WEIGHT_PLACES = Decimal("0.000001") +"""Weights and drifts are fractions; six places is finer than any band anyone sets.""" + +DEFAULT_DIMENSION = AllocationDimension.asset_class + + +@dataclass(frozen=True) +class Position: + """A priced holding the rebalancer may trade.""" + + instrument_id: int + ticker: str + name: str + lot: int + qty: Decimal + """Units held, always positive here — a short is never a rebalancing candidate.""" + unit_value_rub: Decimal + """Clean price of one unit in RUB. Taken from the holding's own valuation rather than + re-converted, so this screen and the portfolio screen cannot disagree; the accrued + interest of a bond is excluded, because it is not part of what a lot costs to buy.""" + price: Decimal + price_currency: str + + @property + def value_rub(self) -> Decimal: + return self.qty * self.unit_value_rub + + +@dataclass(frozen=True) +class Target: + weight: Decimal + band: Decimal = ZERO + + +@dataclass(frozen=True) +class Trade: + """One proposed order. `qty` is whole lots and within both the position and the cash.""" + + instrument_id: int + ticker: str + name: str + action: str + """buy | sell""" + qty: Decimal + lot: int + price: Decimal + price_currency: str + amount_rub: Decimal + """Magnitude of the order in RUB, always positive.""" + blocked_by_cash: bool = False + current_value_rub: Decimal = ZERO + current_weight: Decimal = ZERO + + +@dataclass(frozen=True) +class BucketPlan: + bucket: str + current_value_rub: Decimal + current_weight: Decimal + target_weight: Decimal | None + drift: Decimal | None + within_band: bool + delta_value_rub: Decimal + """Positive: this bucket should grow by this much. Negative: shrink.""" + trades: list[Trade] = field(default_factory=list) + + +@dataclass(frozen=True) +class RebalancePlan: + portfolio_id: int + dimension: AllocationDimension + as_of: date + total_value_rub: Decimal + cash_available_rub: Decimal + buckets: list[BucketPlan] + warnings: list[str] = field(default_factory=list) + + +def _q(value: Decimal) -> Decimal: + return value.quantize(WEIGHT_PLACES) + + +def _floor_lots(qty: Decimal, lot: int) -> Decimal: + """Largest whole number of lots that fits in `qty`, as a quantity.""" + if lot <= 0 or qty <= ZERO: + return ZERO + lots = (qty / Decimal(lot)).to_integral_value(rounding=ROUND_FLOOR) + return max(ZERO, lots) * Decimal(lot) + + +def build_plan( + *, + portfolio_id: int, + dimension: AllocationDimension, + as_of: date, + total_value_rub: Decimal, + bucket_values: Mapping[str, Decimal], + positions: Mapping[str, Sequence[Position]], + targets: Mapping[str, Target], + cash_available_rub: Decimal, + warnings: Sequence[str] = (), +) -> RebalancePlan: + """The whole decision, with no database in sight — see the module docstring for the rules.""" + names = sorted(set(bucket_values) | set(targets)) + plans: list[BucketPlan] = [] + notes = list(warnings) + + drafts: list[BucketPlan] = [] + for name in names: + current = bucket_values.get(name, ZERO) + weight = _q(current / total_value_rub) if total_value_rub > ZERO else ZERO + target = targets.get(name) + if target is None: + drafts.append( + BucketPlan( + bucket=name, + current_value_rub=current, + current_weight=weight, + target_weight=None, + drift=None, + within_band=False, + delta_value_rub=ZERO, + ) + ) + continue + drift = _q(weight - target.weight) + within = abs(drift) <= target.band + delta = ZERO if within else (target.weight - weight) * total_value_rub + drafts.append( + BucketPlan( + bucket=name, + current_value_rub=current, + current_weight=weight, + target_weight=target.weight, + drift=drift, + within_band=within, + delta_value_rub=delta, + ) + ) + + # sells first, so the screen reads in the order the money moves; they fund nothing here + for draft in drafts: + if draft.delta_value_rub < ZERO: + draft.trades.extend(_sell_side(draft, positions.get(draft.bucket, ()), notes)) + + cash_left = cash_available_rub + for draft in sorted(drafts, key=lambda b: -b.delta_value_rub): + if draft.delta_value_rub <= ZERO: + continue + trades, cash_left = _buy_side(draft, positions.get(draft.bucket, ()), cash_left, notes) + draft.trades.extend(trades) + + plans = sorted(drafts, key=lambda b: (-b.current_value_rub, b.bucket)) + return RebalancePlan( + portfolio_id=portfolio_id, + dimension=dimension, + as_of=as_of, + total_value_rub=total_value_rub, + cash_available_rub=cash_available_rub, + buckets=plans, + warnings=notes, + ) + + +def _shares(bucket: BucketPlan, positions: Sequence[Position]) -> list[tuple[Position, Decimal]]: + """Each position and the slice of the bucket's move it carries, largest first.""" + base = sum((p.value_rub for p in positions), start=ZERO) + if base <= ZERO: + return [] + amount = abs(bucket.delta_value_rub) + rows = [(p, amount * p.value_rub / base) for p in positions] + return sorted(rows, key=lambda row: (-row[1], row[0].instrument_id)) + + +def _sell_side(bucket: BucketPlan, positions: Sequence[Position], notes: list[str]) -> list[Trade]: + rows = _shares(bucket, positions) + if not rows: + if bucket.bucket != CASH: + notes.append( + f"бакет «{bucket.bucket}» надо уменьшить, но продавать нечего: " + f"ни одной позиции с ценой" + ) + return [] + out: list[Trade] = [] + for position, amount in rows: + want = _floor_lots(amount / position.unit_value_rub, position.lot) + # never past what is held, and the cap itself is whole lots: no accidental short + qty = min(want, _floor_lots(position.qty, position.lot)) + if qty <= ZERO: + continue + out.append( + Trade( + instrument_id=position.instrument_id, + ticker=position.ticker, + name=position.name, + action="sell", + qty=qty, + lot=position.lot, + price=position.price, + price_currency=position.price_currency, + amount_rub=qty * position.unit_value_rub, + current_value_rub=position.value_rub, + current_weight=bucket.current_weight, + ) + ) + if not out: + notes.append( + f"бакет «{bucket.bucket}» перевешен, но продажа меньше одного лота — рекомендации нет" + ) + return out + + +def _buy_side( + bucket: BucketPlan, positions: Sequence[Position], cash: Decimal, notes: list[str] +) -> tuple[list[Trade], Decimal]: + rows = _shares(bucket, positions) + if not rows: + if bucket.bucket != CASH: + notes.append( + f"бакет «{bucket.bucket}» надо увеличить, но покупать нечего: " + f"ни одной позиции с ценой" + ) + return [], cash + out: list[Trade] = [] + left = cash + for position, amount in rows: + want = _floor_lots(amount / position.unit_value_rub, position.lot) + if want <= ZERO: + continue + affordable = _floor_lots(left / position.unit_value_rub, position.lot) + blocked = affordable < want + qty = min(want, affordable) + if qty <= ZERO and not blocked: + continue + cost = qty * position.unit_value_rub + left -= cost + out.append( + Trade( + instrument_id=position.instrument_id, + ticker=position.ticker, + name=position.name, + action="buy", + qty=qty, + lot=position.lot, + price=position.price, + price_currency=position.price_currency, + amount_rub=cost, + blocked_by_cash=blocked, + current_value_rub=position.value_rub, + current_weight=bucket.current_weight, + ) + ) + if any(t.blocked_by_cash for t in out): + notes.append(f"на докупку бакета «{bucket.bucket}» не хватает свободных денег") + return out, left + + +# --------------------------------------------------------------------------- loading + + +async def _load_positions( + session: AsyncSession, scope: str, dimension: AllocationDimension +) -> tuple[dict[str, list[Position]], list[str]]: + """Priced long positions of the scope, grouped into the dimension's buckets.""" + rows = ( + await session.execute( + select( + MetricHolding.instrument_id, + MetricHolding.qty, + MetricHolding.value_rub, + MetricHolding.accrued_interest_rub, + MetricHolding.market_price, + MetricHolding.price_currency, + Instrument.ticker, + Instrument.name, + Instrument.lot, + Instrument.asset_class, + Instrument.sector, + Instrument.country, + Instrument.currency, + ) + .join(Instrument, Instrument.id == MetricHolding.instrument_id) + .where(MetricHolding.scope == scope) + ) + ).all() + + out: dict[str, list[Position]] = defaultdict(list) + unpriced: list[str] = [] + for r in rows: + qty = Decimal(r.qty or 0) + if qty <= ZERO: + continue + if r.value_rub is None or r.market_price is None: + unpriced.append(r.ticker) + continue + clean = Decimal(r.value_rub) - Decimal(r.accrued_interest_rub or 0) + unit = clean / qty + if unit <= ZERO: + unpriced.append(r.ticker) + continue + bucket = bucket_of( + Holding( + instrument_id=r.instrument_id, + value_rub=clean, + asset_class=str(r.asset_class), + sector=r.sector, + country=r.country, + currency=r.currency, + ), + dimension, + ) + out[bucket].append( + Position( + instrument_id=r.instrument_id, + ticker=r.ticker, + name=r.name, + lot=max(1, int(r.lot or 1)), + qty=qty, + unit_value_rub=unit, + price=Decimal(r.market_price), + price_currency=r.price_currency or r.currency, + ) + ) + return dict(out), sorted(set(unpriced)) + + +async def _load_buckets( + session: AsyncSession, scope: str, dimension: AllocationDimension +) -> dict[str, Decimal]: + rows = ( + await session.execute( + select(MetricAllocation.bucket, MetricAllocation.value_rub).where( + MetricAllocation.scope == scope, MetricAllocation.dimension == dimension + ) + ) + ).all() + return {r.bucket: Decimal(r.value_rub) for r in rows} + + +async def load_targets( + session: AsyncSession, portfolio_id: int, dimension: AllocationDimension +) -> dict[str, Target]: + rows = ( + await session.execute( + select(PortfolioTarget).where( + PortfolioTarget.portfolio_id == portfolio_id, + PortfolioTarget.dimension == dimension, + ) + ) + ).scalars() + return { + r.bucket: Target(weight=Decimal(r.target_weight), band=Decimal(r.band or 0)) for r in rows + } + + +async def portfolio_cash_rub(session: AsyncSession, portfolio_id: int) -> Decimal: + """Free money on the portfolio's accounts, converted to RUB at today's rate. + + A currency with no rate today is left out rather than counted at a made-up rate; the + result is a purchasing power that is understated, never overstated, which is the safe + direction for something that caps a buy. + """ + account_ids = set( + ( + await session.execute( + select(PortfolioAccount.account_id).where( + PortfolioAccount.portfolio_id == portfolio_id + ) + ) + ) + .scalars() + .all() + ) + if not account_ids: + return ZERO + balances = await cash_balances(session) + if not balances: + return ZERO + fx = await FxTable.load(session) + as_of = today_local() + total = ZERO + for (account_id, ccy), amount in balances.items(): + if account_id not in account_ids or amount == ZERO: + continue + rub = fx.to_rub(amount, ccy, as_of) + if rub is not None: + total += rub + return total + + +async def compute_rebalance( + session: AsyncSession, + portfolio_id: int, + dimension: AllocationDimension, + *, + cash_available_rub: Decimal | None = None, +) -> RebalancePlan: + """The plan for one portfolio and dimension, computed from the current metric tables. + + Used by both the refresh step and the API, so a what-if with a different cash figure + cannot drift away from what the stored table says. + """ + scope = f"portfolio:{portfolio_id}" + targets = await load_targets(session, portfolio_id, dimension) + bucket_values = await _load_buckets(session, scope, dimension) + positions, unpriced = await _load_positions(session, scope, dimension) + cash = ( + cash_available_rub + if cash_available_rub is not None + else await portfolio_cash_rub(session, portfolio_id) + ) + total = sum((v for v in bucket_values.values() if v > ZERO), start=ZERO) + + warnings: list[str] = [] + if unpriced: + warnings.append( + f"у {len(unpriced)} инструментов нет цены — в рекомендации не вошли: " + + ", ".join(unpriced) + ) + return build_plan( + portfolio_id=portfolio_id, + dimension=dimension, + as_of=today_local(), + total_value_rub=total, + bucket_values=bucket_values, + positions=positions, + targets=targets, + cash_available_rub=cash, + warnings=warnings, + ) + + +# --------------------------------------------------------------------------- refresh step + + +async def rebuild_rebalance(session: AsyncSession) -> None: + """Replace `metric_rebalance` and fill the target columns of `metric_allocation`. + + Only portfolios that actually have targets are touched: without a decision there is + nothing to compare against, and an empty table is the honest representation of that. + """ + await session.execute(delete(MetricRebalance)) + await session.execute(update(MetricAllocation).values(target_weight=None, drift=None)) + + pairs = ( + await session.execute( + select(PortfolioTarget.portfolio_id, PortfolioTarget.dimension).distinct() + ) + ).all() + if not pairs: + return + + rows: list[dict[str, object]] = [] + for portfolio_id, dimension in sorted(pairs, key=lambda p: (p[0], str(p[1]))): + plan = await compute_rebalance(session, portfolio_id, dimension) + scope = f"portfolio:{portfolio_id}" + for bucket in plan.buckets: + if bucket.target_weight is None: + continue + rows.append( + { + "portfolio_id": portfolio_id, + "dimension": dimension, + "bucket": bucket.bucket, + "instrument_id": None, + "current_value_rub": bucket.current_value_rub, + "current_weight": bucket.current_weight, + "target_weight": bucket.target_weight, + "delta_value_rub": bucket.delta_value_rub, + "suggested_qty": None, + "lot": None, + "price": None, + "price_currency": None, + "within_band": bucket.within_band, + "blocked_by_cash": False, + } + ) + await session.execute( + update(MetricAllocation) + .where( + MetricAllocation.scope == scope, + MetricAllocation.dimension == dimension, + MetricAllocation.bucket == bucket.bucket, + ) + .values(target_weight=bucket.target_weight, drift=bucket.drift) + ) + for trade in bucket.trades: + signed = trade.amount_rub if trade.action == "buy" else -trade.amount_rub + rows.append( + { + "portfolio_id": portfolio_id, + "dimension": dimension, + "bucket": bucket.bucket, + "instrument_id": trade.instrument_id, + "current_value_rub": trade.current_value_rub, + "current_weight": trade.current_weight, + "target_weight": None, + "delta_value_rub": signed, + "suggested_qty": trade.qty, + "lot": trade.lot, + "price": trade.price, + "price_currency": trade.price_currency, + "within_band": bucket.within_band, + "blocked_by_cash": trade.blocked_by_cash, + } + ) + for note in plan.warnings: + FINDINGS.add( + "rebalance_incomplete", + "warn", + f"Портфель {portfolio_id}, разрез {dimension.value}: {note}", + ref={"portfolio_id": portfolio_id, "dimension": dimension.value}, + ) + + if rows: + await session.execute(insert(MetricRebalance), rows) + log.info("rebalance: %s portfolio/dimension pairs, %s rows", len(pairs), len(rows)) diff --git a/backend/src/fintracker/analytics/tax.py b/backend/src/fintracker/analytics/tax.py new file mode 100644 index 0000000..a0575f2 --- /dev/null +++ b/backend/src/fintracker/analytics/tax.py @@ -0,0 +1,402 @@ +"""The tax picture per calendar year and account — an estimate, labelled one (plan §3, §4). + +**This module never produces an authoritative number and must never be presented as one.** +The broker is the tax agent: it computes, withholds and files. What this exists for is two +things the broker's certificate cannot do — let the certificate be *checked* against the +ledger the app already has, and show, before a sale happens, what selling a position ahead +of the three-year mark would cost. + +What goes into the numbers, and why: + +* **Dividends and coupons** are taken gross. `event.amount` is the cash that actually landed + and `event.tax` is what the broker held back on the way (ledger.py: both are stored so + that `amount + tax` is the gross), so the pair reconstructs the payment without a second + source. Standalone `tax` events are deliberately NOT added on top: for every feed the + project reads, the withholding is reported inside the payment operation, and counting both + would double it. If a broker ever reports it separately, that shows up as a mismatch + against the certificate — which is the whole point of the screen. +* **Realised results** come from `lot_disposal.realized_pnl_rub`, which `ledger/rebuild.py` + computed as `proceeds_rub - cost_rub` with **each leg converted at the CBR rate of its own + date** — the purchase at the purchase date, the sale at the sale date. That is not a + rounding choice, it is the law: for a foreign-currency security the rouble revaluation is + part of the base (plan §7, вопрос 4), so a paper that did not move in dollars still + realises a rouble gain when the dollar rose. The exception the plan names — Ministry of + Finance eurobonds, where revaluation is excluded — is **not** implemented, because nothing + in the instrument master distinguishes one reliably (`issuer` is free text and mostly + empty). Instead, every foreign-currency disposal is counted and reported as a finding, so + a eurobond among them can be spotted and corrected by hand rather than being silently + mis-taxed in either direction. +* **A disposal with no rate on one of its legs has `realized_pnl_rub = NULL`** and is left + out of the totals entirely. It is never substituted with the other leg's rate or with + today's: the count goes to `FINDINGS`, so the year is visibly incomplete instead of quietly + wrong. +* **ЛДВ (art. 219.1)** — three full years on an exchange-traded instrument. The flag is + `lot_disposal.ldv_eligible`, produced by the single rule in `ledger/lots.py`; this module + reads it and does not restate it. It is an estimate for the reason the plan records as + вопрос 3: the classic three-year rule is applied to MOEX-traded papers only, and the 2025 + changes around foreign issuers and ИИС-3 are out of scope. An ИИС is an ordinary `account` + here, so its own regime is not modelled at all. +* **The rate is stored, not baked in.** `TAX_RATE` is a constant of this module, but it is + written into every `metric_tax_year` row, so a year computed under one rate stays readable + after the constant changes, and a changed rate is visible rather than invisible. + +The base follows the shape the phase-4 contract fixes: + + taxable_base = max(0, realized_gain + realized_loss - ldv_exempt) + +Gains and losses are both over all disposals of the year (the loss is negative), and +`ldv_exempt` is the gain on the LDV-eligible ones, subtracted back out. Dividends and coupons +are outside the base: the agent withholds on them at source, which is what +`tax_withheld_rub` records. +""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from dataclasses import dataclass, field +from datetime import date, timedelta +from decimal import ROUND_HALF_UP, Decimal + +from sqlalchemy import delete, insert, select +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import FINDINGS, today_local +from fintracker.ledger.lots import LDV_DAYS +from fintracker.ledger.rebuild import EXCHANGE_CLASSES +from fintracker.models import ( + Account, + Event, + EventKind, + EventStatus, + Instrument, + Lot, + LotDisposal, + MetricTaxYear, +) +from fintracker.pricing.fx import FxTable +from fintracker.pricing.prices import PriceTable + +log = logging.getLogger(__name__) + +ZERO = Decimal(0) +MONEY_PLACES = Decimal("0.01") + +TAX_RATE = Decimal("0.13") +"""НДФЛ on investment income for a resident. Written into every row, never assumed by a +reader of the row: see the module docstring.""" + +DISCLAIMER = "Оценка. Налоговый агент — брокер; сверяйтесь с его справкой." +"""Shown next to every number this module produces, on every screen, without exception.""" + +INCOME_KINDS = (EventKind.dividend, EventKind.coupon) + +RUB = "RUB" + + +def money(value: Decimal) -> Decimal: + return value.quantize(MONEY_PLACES, rounding=ROUND_HALF_UP) + + +def tax_on(base: Decimal, rate: Decimal = TAX_RATE) -> Decimal: + """Tax on a base, floored at zero — a negative base is a loss, not a refund.""" + return money(base * rate) if base > ZERO else money(ZERO) + + +def ldv_date_for(open_date: date) -> date: + """First day a sale of this lot qualifies for the long-term exemption.""" + return open_date + timedelta(days=LDV_DAYS) + + +# -------------------------------------------------------------------------------------- +# refresh step +# -------------------------------------------------------------------------------------- + + +@dataclass +class _Bucket: + """One (year, account) cell while it is being filled.""" + + dividends: Decimal = ZERO + coupons: Decimal = ZERO + withheld: Decimal = ZERO + gain: Decimal = ZERO + loss: Decimal = ZERO + ldv_exempt: Decimal = ZERO + + +@dataclass +class _Gaps: + """What had to be left out, so the totals can say so instead of pretending.""" + + income_no_fx: int = 0 + disposals_no_fx: int = 0 + foreign_disposals: int = 0 + foreign_currencies: set[str] = field(default_factory=set) + + +async def rebuild_tax_year(session: AsyncSession) -> None: + """Replace `metric_tax_year` for every year and account the ledger touches.""" + await session.execute(delete(MetricTaxYear)) + + fx = await FxTable.load(session) + buckets: dict[tuple[int, int], _Bucket] = defaultdict(_Bucket) + gaps = _Gaps() + + await _add_income(session, fx, buckets, gaps) + await _add_realized(session, buckets, gaps) + + rows = [] + for (year, account_id), b in sorted(buckets.items()): + base = b.gain + b.loss - b.ldv_exempt + rows.append( + { + "year": year, + "account_id": account_id, + "dividends_gross_rub": money(b.dividends), + "coupons_gross_rub": money(b.coupons), + "tax_withheld_rub": money(b.withheld), + "realized_gain_rub": money(b.gain), + "realized_loss_rub": money(b.loss), + "ldv_exempt_rub": money(b.ldv_exempt), + "taxable_base_rub": money(base if base > ZERO else ZERO), + "estimated_tax_rub": tax_on(base), + "tax_rate": TAX_RATE, + } + ) + if rows: + await session.execute(insert(MetricTaxYear), rows) + _report(gaps) + log.info("tax: %s (year, account) rows", len(rows)) + + +async def _add_income( + session: AsyncSession, + fx: FxTable, + buckets: dict[tuple[int, int], _Bucket], + gaps: _Gaps, +) -> None: + """Dividends and coupons, gross, with the tax the broker already took.""" + rows = ( + await session.execute( + select( + Event.account_id, + Event.trade_date, + Event.kind, + Event.amount, + Event.currency, + Event.tax, + Event.tax_currency, + ).where(Event.status == EventStatus.confirmed, Event.kind.in_(INCOME_KINDS)) + ) + ).all() + for r in rows: + withheld = Decimal(r.tax or 0) + gross = Decimal(r.amount or 0) + withheld + gross_rub = fx.to_rub(gross, r.currency, r.trade_date) + withheld_rub = fx.to_rub(withheld, r.tax_currency or r.currency, r.trade_date) + if gross_rub is None or withheld_rub is None: + gaps.income_no_fx += 1 + continue + bucket = buckets[(r.trade_date.year, r.account_id)] + if r.kind == EventKind.dividend: + bucket.dividends += gross_rub + else: + bucket.coupons += gross_rub + bucket.withheld += withheld_rub + + +async def _add_realized( + session: AsyncSession, buckets: dict[tuple[int, int], _Bucket], gaps: _Gaps +) -> None: + """FIFO results, already in roubles at each leg's own date (`ledger/rebuild.py`).""" + rows = ( + await session.execute( + select( + Lot.account_id, + Lot.cost_currency, + LotDisposal.close_date, + LotDisposal.proceeds_currency, + LotDisposal.realized_pnl_rub, + LotDisposal.ldv_eligible, + ).join(Lot, Lot.id == LotDisposal.lot_id) + ) + ).all() + for r in rows: + if r.realized_pnl_rub is None: + gaps.disposals_no_fx += 1 + continue + for ccy in (r.cost_currency, r.proceeds_currency): + if ccy and ccy.upper() != RUB: + gaps.foreign_disposals += 1 + gaps.foreign_currencies.add(ccy.upper()) + break + pnl = Decimal(r.realized_pnl_rub) + bucket = buckets[(r.close_date.year, r.account_id)] + if pnl >= ZERO: + bucket.gain += pnl + if r.ldv_eligible: + bucket.ldv_exempt += pnl + else: + bucket.loss += pnl + if r.ldv_eligible: + # a loss on an exempt lot is not deductible either — art. 219.1 removes the + # whole result from the base, not only the profitable half + bucket.ldv_exempt += pnl + + +def _report(gaps: _Gaps) -> None: + if gaps.income_no_fx: + FINDINGS.add( + "tax_income_no_fx", + "warn", + f"{gaps.income_no_fx} выплат без курса ЦБ на дату — в налоговый год не вошли", + count=gaps.income_no_fx, + ) + if gaps.disposals_no_fx: + FINDINGS.add( + "tax_disposal_no_fx", + "warn", + f"{gaps.disposals_no_fx} закрытий лотов без курса на одну из ног — " + "рублёвый результат неизвестен, в базу не вошли", + count=gaps.disposals_no_fx, + ) + if gaps.foreign_disposals: + FINDINGS.add( + "tax_currency_revaluation", + "info", + f"{gaps.foreign_disposals} закрытий в валюте " + f"({', '.join(sorted(gaps.foreign_currencies))}): валютная переоценка включена " + "в базу. Еврооблигации Минфина, где она не облагается, автоматически не " + "распознаются — проверьте вручную", + count=gaps.foreign_disposals, + ) + + +# -------------------------------------------------------------------------------------- +# open lots: the screen that shows the cost of selling early +# -------------------------------------------------------------------------------------- + + +@dataclass(frozen=True) +class OpenLotTax: + """One open lot and what selling it today would cost — all of it an estimate.""" + + lot_id: int + instrument_id: int + ticker: str | None + name: str + account_id: int + account_name: str + open_date: date + qty_remaining: Decimal + cost_rub: Decimal | None + market_value_rub: Decimal | None + unrealized_gain_rub: Decimal | None + ldv_eligible: bool + ldv_date: date | None + days_to_ldv: int | None + tax_if_sold_now_rub: Decimal | None + + +async def open_lot_tax( + session: AsyncSession, + *, + as_of: date | None = None, + account_id: int | None = None, + rate: Decimal = TAX_RATE, +) -> list[OpenLotTax]: + """Every long lot still open, with its ЛДВ date and the tax a sale today would trigger. + + Read live rather than from a metric table: it depends on today's date and today's price, + and a table refreshed nightly would show yesterday's `days_to_ldv` — off by one on + exactly the day the answer matters. + + Short lots are excluded. `qty_remaining` is stored signed, and a short position has no + holding period to accrue: art. 219.1 is about owning a paper for three years. + + `cost_rub` is the lot's cost at the rate of ITS OWN open date, prorated to the part still + held — the same convention the realised numbers use, so an open lot and the closure it + later becomes are measured the same way. `market_value_rub` is at today's rate, which is + what makes the difference a currency-revaluation figure rather than a price-only one. + """ + as_of = as_of or today_local() + conditions = [Lot.qty_remaining > 0] + if account_id is not None: + conditions.append(Lot.account_id == account_id) + rows = ( + await session.execute( + select( + Lot.id, + Lot.account_id, + Lot.instrument_id, + Lot.open_date, + Lot.qty_open, + Lot.qty_remaining, + Lot.cost_total_rub, + Account.name.label("account_name"), + Instrument.ticker, + Instrument.name.label("instrument_name"), + Instrument.asset_class, + Instrument.board, + ) + .join(Account, Account.id == Lot.account_id) + .join(Instrument, Instrument.id == Lot.instrument_id) + .where(*conditions) + .order_by(Lot.open_date, Lot.id) + ) + ).all() + + fx = await FxTable.load(session) + prices = await PriceTable.load(session) + + out: list[OpenLotTax] = [] + for r in rows: + qty = Decimal(r.qty_remaining) + cost_rub = None + if r.cost_total_rub is not None and r.qty_open: + cost_rub = money(Decimal(r.cost_total_rub) * qty / Decimal(r.qty_open)) + + quote = prices.latest(r.instrument_id, as_of) + value_rub = None + if quote is not None: + value_rub = fx.to_rub(qty * quote.total, quote.currency, as_of) + if value_rub is not None: + value_rub = money(value_rub) + + unrealized = None + if cost_rub is not None and value_rub is not None: + unrealized = money(value_rub - cost_rub) + + exchange_traded = str(r.asset_class) in EXCHANGE_CLASSES and bool(r.board) + ldv_date = ldv_date_for(r.open_date) if exchange_traded else None + eligible = ldv_date is not None and as_of >= ldv_date + days_to_ldv = None if ldv_date is None else max((ldv_date - as_of).days, 0) + + if eligible: + tax_now: Decimal | None = money(ZERO) + elif unrealized is None: + tax_now = None + else: + tax_now = tax_on(unrealized, rate) + + out.append( + OpenLotTax( + lot_id=r.id, + instrument_id=r.instrument_id, + ticker=r.ticker, + name=r.instrument_name, + account_id=r.account_id, + account_name=r.account_name, + open_date=r.open_date, + qty_remaining=qty, + cost_rub=cost_rub, + market_value_rub=value_rub, + unrealized_gain_rub=unrealized, + ldv_eligible=eligible, + ldv_date=ldv_date, + days_to_ldv=days_to_ldv, + tax_if_sold_now_rub=tax_now, + ) + ) + return out diff --git a/backend/src/fintracker/api/app.py b/backend/src/fintracker/api/app.py index 719f62a..d28731f 100644 --- a/backend/src/fintracker/api/app.py +++ b/backend/src/fintracker/api/app.py @@ -18,17 +18,22 @@ from fintracker.api.routers import ( accounts, analytics, auth, + benchmarks, cashflow, categories, events, + goals, health, imports, + income, instruments, links, metrics, networth, + rebalance, rules, sync, + tax, transactions, ) from fintracker.api.web import mount_web @@ -95,6 +100,12 @@ def create_app() -> FastAPI: app.include_router(instruments.router, prefix=API_PREFIX) app.include_router(links.router, prefix=API_PREFIX) app.include_router(metrics.router, prefix=API_PREFIX) + app.include_router(goals.router, prefix=API_PREFIX) + app.include_router(income.router, prefix=API_PREFIX) + app.include_router(rebalance.router, prefix=API_PREFIX) + app.include_router(tax.router, prefix=API_PREFIX) + app.include_router(benchmarks.router, prefix=API_PREFIX) + app.include_router(benchmarks.analytics_router, prefix=API_PREFIX) if settings.web_dir is not None: mount_web(app, settings.web_dir, API_PREFIX) return app diff --git a/backend/src/fintracker/api/routers/benchmarks.py b/backend/src/fintracker/api/routers/benchmarks.py new file mode 100644 index 0000000..5a7e80a --- /dev/null +++ b/backend/src/fintracker/api/routers/benchmarks.py @@ -0,0 +1,232 @@ +"""Benchmarks: the list the user maintains, and the comparison built from it. + +Two routers live here because the two halves belong to two prefixes the contract fixes: +`router` serves `/benchmarks` (CRUD on the user's choice of indices) and `analytics_router` +serves `/analytics/benchmarks` (the read of `metric_benchmark_returns` beside +`metric_returns`). Both are exported for `api/app.py` to include. + +The comparison itself is not computed here — `analytics/benchmarks.py` built it during the +refresh, on the portfolio's own date grid. Only `excess` is derived per request, as a +subtraction of two numbers that are already on the same grid. +""" + +from __future__ import annotations + +from datetime import date +from typing import Annotated + +from fastapi import APIRouter, Query, Response, status +from sqlalchemy import delete, func, select + +from fintracker.api.deps import CurrentUser, SessionDep +from fintracker.api.errors import Problem +from fintracker.api.schemas.benchmarks import ( + BenchmarkComparison, + BenchmarkComparisonOut, + BenchmarkCreate, + BenchmarkOut, + BenchmarkPatch, + BenchmarkReturnOut, +) +from fintracker.api.scopes import DEFAULT_SCOPE, resolve_scope +from fintracker.models import ( + Benchmark, + BenchmarkKind, + MetricBenchmarkReturns, + MetricReturns, + PriceDaily, +) + +router = APIRouter(prefix="/benchmarks", tags=["benchmarks"]) +analytics_router = APIRouter(prefix="/analytics", tags=["benchmarks"]) + +#: Order the periods are shown in; the tables store them unordered. +PERIOD_ORDER = {p: i for i, p in enumerate(("1m", "3m", "6m", "ytd", "1y", "3y", "all"))} + +ScopeParam = Annotated[str, Query(description="all | account: | portfolio:")] + + +def _kind(value: str) -> BenchmarkKind: + try: + return BenchmarkKind(value) + except ValueError: + allowed = ", ".join(k.value for k in BenchmarkKind) + raise Problem( + status.HTTP_422_UNPROCESSABLE_ENTITY, "Unprocessable", f"kind: ожидается {allowed}" + ) from None + + +async def _history(session: SessionDep) -> dict[int, tuple[date, date]]: + """The price history each benchmark instrument actually has, for the list response.""" + rows = ( + await session.execute( + select( + PriceDaily.instrument_id, func.min(PriceDaily.d), func.max(PriceDaily.d) + ).group_by(PriceDaily.instrument_id) + ) + ).all() + return {r[0]: (r[1], r[2]) for r in rows} + + +def _out(benchmark: Benchmark, history: dict[int, tuple[date, date]]) -> BenchmarkOut: + span = history.get(benchmark.instrument_id or 0) + return BenchmarkOut( + id=benchmark.id, + code=benchmark.code, + name=benchmark.name, + kind=str(benchmark.kind), + source=benchmark.source, + currency=benchmark.currency, + is_default=benchmark.is_default, + is_active=benchmark.is_active, + instrument_id=benchmark.instrument_id, + history_from=span[0] if span else None, + history_to=span[1] if span else None, + ) + + +@router.get("", name="list") +async def list_benchmarks(session: SessionDep, _: CurrentUser) -> list[BenchmarkOut]: + """Every benchmark, defaults first — the order the comparison block shows them in.""" + found = await session.execute( + select(Benchmark).order_by(Benchmark.is_default.desc(), Benchmark.code) + ) + history = await _history(session) + return [_out(b, history) for b in found.scalars()] + + +@router.post("", name="create", status_code=status.HTTP_201_CREATED) +async def create_benchmark( + body: BenchmarkCreate, session: SessionDep, _: CurrentUser +) -> BenchmarkOut: + existing = ( + await session.execute(select(Benchmark).where(Benchmark.code == body.code)) + ).scalar_one_or_none() + if existing is not None: + raise Problem(status.HTTP_409_CONFLICT, "Conflict", f"Бенчмарк {body.code} уже есть") + data = body.model_dump() + data["kind"] = _kind(body.kind) + benchmark = Benchmark(**data) + session.add(benchmark) + await session.commit() + await session.refresh(benchmark) + return _out(benchmark, await _history(session)) + + +@router.patch("/{benchmark_id}", name="patch") +async def patch_benchmark( + benchmark_id: int, body: BenchmarkPatch, session: SessionDep, _: CurrentUser +) -> BenchmarkOut: + benchmark = await session.get(Benchmark, benchmark_id) + if benchmark is None: + raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет бенчмарка {benchmark_id}") + changes = body.model_dump(exclude_unset=True) + for field in ("code", "name", "kind", "source", "currency", "is_default", "is_active"): + if field in changes and changes[field] is None: + raise Problem(status.HTTP_400_BAD_REQUEST, "Bad request", f"{field} не может быть null") + if "kind" in changes: + changes["kind"] = _kind(changes["kind"]) + for field, value in changes.items(): + setattr(benchmark, field, value) + await session.commit() + await session.refresh(benchmark) + return _out(benchmark, await _history(session)) + + +@router.delete("/{benchmark_id}", name="delete", status_code=status.HTTP_204_NO_CONTENT) +async def delete_benchmark(benchmark_id: int, session: SessionDep, _: CurrentUser) -> Response: + benchmark = await session.get(Benchmark, benchmark_id) + if benchmark is None: + raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет бенчмарка {benchmark_id}") + # the metric rows go with it: a comparison against an index nobody tracks any more is + # not a number the screens should still be able to find + await session.execute( + delete(MetricBenchmarkReturns).where(MetricBenchmarkReturns.benchmark_id == benchmark_id) + ) + await session.delete(benchmark) + await session.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@analytics_router.get("/benchmarks", name="benchmarks") +async def compare( + session: SessionDep, + _: CurrentUser, + scope: ScopeParam = DEFAULT_SCOPE, + period: Annotated[ + list[str] | None, Query(description="repeatable: 1m 3m 6m ytd 1y 3y all") + ] = None, +) -> BenchmarkComparison: + """The portfolio's TWR beside each benchmark's, on the same grid of days. + + `days_skipped` is returned on both sides and is not cosmetic: a non-zero value on either + means the two chains did not cover the same days, and the difference is then an + approximation of an excess return rather than one. + """ + await resolve_scope(session, scope) + conditions = [MetricReturns.scope == scope] + if period: + conditions.append(MetricReturns.period.in_(period)) + portfolio = ( + await session.execute( + select( + MetricReturns.period, + MetricReturns.date_from, + MetricReturns.date_to, + MetricReturns.twr, + MetricReturns.twr_annualized, + MetricReturns.twr_days_skipped, + ).where(*conditions) + ) + ).all() + + rows = ( + await session.execute( + select( + MetricBenchmarkReturns.period, + MetricBenchmarkReturns.twr, + MetricBenchmarkReturns.twr_annualized, + MetricBenchmarkReturns.days_skipped, + Benchmark.id, + Benchmark.code, + Benchmark.name, + Benchmark.kind, + ) + .join(Benchmark, Benchmark.id == MetricBenchmarkReturns.benchmark_id) + .where(MetricBenchmarkReturns.scope == scope) + .order_by(Benchmark.is_default.desc(), Benchmark.code) + ) + ).all() + by_period: dict[str, list[BenchmarkReturnOut]] = {} + for r in rows: + by_period.setdefault(r.period, []).append( + BenchmarkReturnOut( + benchmark_id=r.id, + code=r.code, + name=r.name, + kind=str(r.kind), + twr=r.twr, + twr_annualized=r.twr_annualized, + days_skipped=r.days_skipped, + excess=None, + ) + ) + + out = [] + for p in sorted(portfolio, key=lambda r: PERIOD_ORDER.get(r.period, 99)): + benchmarks = [] + for b in by_period.get(p.period, []): + excess = None if p.twr is None or b.twr is None else p.twr - b.twr + benchmarks.append(b.model_copy(update={"excess": excess})) + out.append( + BenchmarkComparisonOut( + period=p.period, + date_from=p.date_from, + date_to=p.date_to, + portfolio_twr=p.twr, + portfolio_twr_annualized=p.twr_annualized, + portfolio_days_skipped=p.twr_days_skipped, + benchmarks=benchmarks, + ) + ) + return BenchmarkComparison(rows=out) diff --git a/backend/src/fintracker/api/routers/goals.py b/backend/src/fintracker/api/routers/goals.py new file mode 100644 index 0000000..a420492 --- /dev/null +++ b/backend/src/fintracker/api/routers/goals.py @@ -0,0 +1,113 @@ +"""Goals: CRUD plus the derived progress (docs/ai/phase4-contract.md §4). + +Progress is computed per request through `analytics/goals.compute_goal_progress` for the +same reason the rebalancing endpoint does: a goal created a minute ago has no row in +`metric_goal_progress` yet, and «no data» for a goal the user just typed in reads as a bug. +The refresh step fills the table from the identical function, so the screen and the stored +metric cannot drift apart. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Response, status +from sqlalchemy import select + +from fintracker.analytics.goals import compute_goal_progress +from fintracker.api.deps import CurrentUser, SessionDep +from fintracker.api.errors import Problem +from fintracker.api.schemas.goals import GoalCreate, GoalOut, GoalPatch, GoalProgressOut +from fintracker.api.scopes import resolve_scope +from fintracker.models import Goal + +router = APIRouter(prefix="/goals", tags=["goals"]) + + +async def _goal(session: SessionDep, goal_id: int) -> Goal: + goal = await session.get(Goal, goal_id) + if goal is None: + raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет цели с id {goal_id}") + return goal + + +async def _check_name(session: SessionDep, name: str, *, exclude: int | None = None) -> None: + stmt = select(Goal.id).where(Goal.name == name) + if exclude is not None: + stmt = stmt.where(Goal.id != exclude) + if (await session.execute(stmt)).scalar_one_or_none() is not None: + raise Problem(status.HTTP_409_CONFLICT, "Conflict", f"Цель «{name}» уже есть") + + +@router.get("", name="list") +async def list_goals( + session: SessionDep, _: CurrentUser, include_archived: bool = False +) -> list[GoalOut]: + stmt = select(Goal).order_by(Goal.id) + if not include_archived: + stmt = stmt.where(Goal.archived.is_(False)) + rows = (await session.execute(stmt)).scalars().all() + return [GoalOut.model_validate(g, from_attributes=True) for g in rows] + + +@router.post("", name="create", status_code=status.HTTP_201_CREATED) +async def create_goal(body: GoalCreate, session: SessionDep, _: CurrentUser) -> GoalOut: + await _check_name(session, body.name) + # a goal pointed at a scope the metrics never built would silently read as zero forever + await resolve_scope(session, body.scope) + goal = Goal(**body.model_dump()) + session.add(goal) + await session.commit() + await session.refresh(goal) + return GoalOut.model_validate(goal, from_attributes=True) + + +@router.patch("/{goal_id}", name="patch") +async def patch_goal(goal_id: int, body: GoalPatch, session: SessionDep, _: CurrentUser) -> GoalOut: + goal = await _goal(session, goal_id) + changes = body.model_dump(exclude_unset=True) + for field in ("name", "scope", "target_amount", "currency", "archived"): + if field in changes and changes[field] is None: + raise Problem( + status.HTTP_400_BAD_REQUEST, "Bad request", f"Поле {field} не может быть null" + ) + if "name" in changes: + await _check_name(session, changes["name"], exclude=goal_id) + if "scope" in changes: + await resolve_scope(session, changes["scope"]) + for field, value in changes.items(): + setattr(goal, field, value) + await session.commit() + await session.refresh(goal) + return GoalOut.model_validate(goal, from_attributes=True) + + +@router.delete("/{goal_id}", name="delete", status_code=status.HTTP_204_NO_CONTENT) +async def delete_goal(goal_id: int, session: SessionDep, _: CurrentUser) -> Response: + goal = await _goal(session, goal_id) + await session.delete(goal) + await session.commit() + return Response(status_code=status.HTTP_204_NO_CONTENT) + + +@router.get("/{goal_id}/progress", name="progress") +async def goal_progress(goal_id: int, session: SessionDep, _: CurrentUser) -> GoalProgressOut: + goal = await _goal(session, goal_id) + progress = await compute_goal_progress(session, goal) + if progress is None: + raise Problem( + status.HTTP_409_CONFLICT, + "Conflict", + f"Прогресс цели «{goal.name}» не считается: нет курса для {goal.currency} " + f"или сумма цели неположительна", + ) + return GoalProgressOut( + goal_id=progress.goal_id, + as_of=progress.as_of, + current_value_rub=progress.current_value_rub, + target_amount_rub=progress.target_amount_rub, + progress=progress.progress, + projected_date=progress.projected_date, + basis=progress.basis, + assumed_rate=progress.assumed_rate, + monthly_needed_rub=progress.monthly_needed_rub, + on_track=progress.on_track, + ) diff --git a/backend/src/fintracker/api/routers/income.py b/backend/src/fintracker/api/routers/income.py new file mode 100644 index 0000000..dae03f8 --- /dev/null +++ b/backend/src/fintracker/api/routers/income.py @@ -0,0 +1,288 @@ +"""`/income` — the dividend and coupon calendar, its history and its forecast (contract §1). + +Every endpoint is a read of `metric_income_calendar` / `metric_income_monthly`, which +`analytics/income.py` rebuilt during the last refresh. Nothing is computed per request except +the grouping the screen asks for, so the three tabs of the screen cannot disagree. + +The one rule worth repeating here: **future rows are split by `basis` wherever they are +summed.** `total_expected_rub` and each month's `amount_rub` are still handed over as one +number because the screen needs one, but `by_basis` sits next to it every time, so a total +that is mostly `history` can be recognised as mostly guesswork. +""" + +from __future__ import annotations + +import logging +from collections import defaultdict +from datetime import date, timedelta +from decimal import Decimal +from typing import Annotated + +from fastapi import APIRouter, Query +from sqlalchemy import select + +from fintracker.analytics import today_local +from fintracker.analytics.income import add_months, month_start +from fintracker.api.deps import CurrentUser, SessionDep +from fintracker.api.schemas.income import ( + CalendarEntry, + CalendarOut, + ForecastMonth, + ForecastOut, + HistoryOut, + HistoryRow, + HistoryTotals, +) +from fintracker.api.scopes import DEFAULT_SCOPE, resolve_scope +from fintracker.models import ( + IncomeBasis, + Instrument, + MetricDataQuality, + MetricIncomeCalendar, + MetricIncomeMonthly, + MetricPortfolioValueDaily, +) + +log = logging.getLogger(__name__) + +router = APIRouter(prefix="/income", tags=["income"]) + +ZERO = Decimal(0) +RUB = "RUB" + +DEFAULT_FORECAST_MONTHS = 12 + +#: Findings `analytics/income.py` raises; the forecast repeats them so the screen can show +#: what it could not cover instead of silently under-reporting. +INCOME_CHECKS = ("income_without_history", "income_irregular_history", "income_missing_fx") + +ScopeParam = Annotated[str, Query(description="all | account: | portfolio:")] + + +@router.get("/calendar", name="calendar") +async def calendar( + session: SessionDep, + _: CurrentUser, + scope: ScopeParam = DEFAULT_SCOPE, + date_from: Annotated[date | None, Query(alias="date_from")] = None, + date_to: Annotated[date | None, Query(alias="date_to")] = None, + include_paid: bool = False, +) -> CalendarOut: + """Payments in a window, one row each; defaults to the next 12 months, forecast only.""" + await resolve_scope(session, scope) + as_of = today_local() + start = date_from if date_from is not None else as_of + # the default window is the twelve months AHEAD, half-open: a quarterly payer must give + # four entries, and an inclusive year-end boundary would let a fifth in on some dates + end = ( + date_to + if date_to is not None + else add_months(as_of, DEFAULT_FORECAST_MONTHS) - timedelta(days=1) + ) + + conditions = [ + MetricIncomeCalendar.scope == scope, + MetricIncomeCalendar.expected_date >= start, + MetricIncomeCalendar.expected_date <= end, + ] + if not include_paid: + conditions.append(MetricIncomeCalendar.basis != IncomeBasis.paid) + + rows = ( + await session.execute( + select(MetricIncomeCalendar, Instrument) + .join(Instrument, Instrument.id == MetricIncomeCalendar.instrument_id) + .where(*conditions) + .order_by(MetricIncomeCalendar.expected_date, Instrument.ticker) + ) + ).all() + + entries = [_entry(row, instrument) for row, instrument in rows] + by_basis: dict[str, Decimal] = defaultdict(lambda: ZERO) + total = ZERO + for row, _instrument in rows: + if row.basis == IncomeBasis.paid or row.amount_rub is None: + continue + by_basis[str(row.basis.value)] += Decimal(row.amount_rub) + total += Decimal(row.amount_rub) + + return CalendarOut( + as_of=as_of, + currency=RUB, + total_expected_rub=total, + entries=entries, + by_basis=dict(sorted(by_basis.items())), + ) + + +@router.get("/history", name="history") +async def history( + session: SessionDep, + _: CurrentUser, + scope: ScopeParam = DEFAULT_SCOPE, + group: Annotated[str, Query(description="month")] = "month", + date_from: Annotated[date | None, Query(alias="date_from")] = None, + date_to: Annotated[date | None, Query(alias="date_to")] = None, + kind: Annotated[str | None, Query(description="dividend | coupon | …")] = None, +) -> HistoryOut: + """Income actually received, grouped by month, kind and currency. + + `group` exists for the contract's sake and accepts only `month`: the table is stored + monthly, and a finer grouping would have to re-read the ledger, which is what the + calendar's `paid` rows already do per payment. + """ + await resolve_scope(session, scope) + conditions = [MetricIncomeMonthly.scope == scope] + if date_from is not None: + conditions.append(MetricIncomeMonthly.month >= month_start(date_from)) + if date_to is not None: + conditions.append(MetricIncomeMonthly.month <= date_to) + if kind is not None: + conditions.append(MetricIncomeMonthly.kind == kind) + + rows = ( + ( + await session.execute( + select(MetricIncomeMonthly) + .where(*conditions) + .order_by( + MetricIncomeMonthly.month, + MetricIncomeMonthly.kind, + MetricIncomeMonthly.currency, + ) + ) + ) + .scalars() + .all() + ) + + total_rub = ZERO + total_tax_rub = ZERO + for r in rows: + if r.amount_rub is None: + continue + total_rub += Decimal(r.amount_rub) + # the tax is stored native; it converts at the same effective rate the month's own + # amount did, which is exact for a single-payment month and right on average otherwise + if r.tax_withheld and r.amount: + total_tax_rub += Decimal(r.tax_withheld) * Decimal(r.amount_rub) / Decimal(r.amount) + + return HistoryOut( + rows=[ + HistoryRow( + month=r.month, + kind=r.kind, + currency=r.currency, + amount=r.amount, + amount_rub=r.amount_rub, + tax_withheld=r.tax_withheld, + payment_count=r.payment_count, + ) + for r in rows + ], + totals=HistoryTotals(amount_rub=total_rub, tax_withheld_rub=total_tax_rub), + ) + + +@router.get("/forecast", name="forecast") +async def forecast( + session: SessionDep, + _: CurrentUser, + scope: ScopeParam = DEFAULT_SCOPE, + months: Annotated[int, Query(ge=1, le=36)] = DEFAULT_FORECAST_MONTHS, +) -> ForecastOut: + """Expected income per month, split by basis, plus the yield it implies.""" + await resolve_scope(session, scope) + as_of = today_local() + end = add_months(as_of, months) + + rows = ( + ( + await session.execute( + select(MetricIncomeCalendar).where( + MetricIncomeCalendar.scope == scope, + MetricIncomeCalendar.basis != IncomeBasis.paid, + MetricIncomeCalendar.expected_date >= as_of, + # half-open, like the calendar's default window and for the same reason + MetricIncomeCalendar.expected_date < end, + ) + ) + ) + .scalars() + .all() + ) + + buckets: dict[date, dict[str, Decimal]] = defaultdict(lambda: defaultdict(lambda: ZERO)) + total = ZERO + for r in rows: + if r.amount_rub is None: + continue # unconvertible: counted in `income_missing_fx`, not silently as zero + amount = Decimal(r.amount_rub) + buckets[month_start(r.expected_date)][str(r.basis.value)] += amount + total += amount + + out_months = [ + ForecastMonth( + month=month, + amount_rub=sum(by_basis.values(), start=ZERO), + by_basis=dict(sorted(by_basis.items())), + ) + for month, by_basis in sorted(buckets.items()) + ] + + return ForecastOut( + months=out_months, + total_rub=total, + annual_yield_on_value=await _yield_on_value(session, scope, total, months), + warnings=await _warnings(session), + ) + + +def _entry(row: MetricIncomeCalendar, instrument: Instrument) -> CalendarEntry: + return CalendarEntry( + instrument_id=row.instrument_id, + ticker=instrument.ticker, + name=instrument.name, + kind=row.kind, + expected_date=row.expected_date, + record_date=row.record_date, + qty=row.qty, + per_unit=row.per_unit, + amount=row.amount, + currency=row.currency, + amount_rub=row.amount_rub, + basis=str(row.basis.value), + tax_withheld=row.tax_withheld, + ) + + +async def _yield_on_value( + session: SessionDep, scope: str, total: Decimal, months: int +) -> Decimal | None: + """Expected income annualised over the scope's latest total value; None without one.""" + value = ( + await session.execute( + select(MetricPortfolioValueDaily.total_rub) + .where(MetricPortfolioValueDaily.scope == scope) + .order_by(MetricPortfolioValueDaily.d.desc()) + .limit(1) + ) + ).scalar_one_or_none() + if value is None or Decimal(value) <= ZERO or months <= 0: + return None + return total * Decimal(12) / Decimal(months) / Decimal(value) + + +async def _warnings(session: SessionDep) -> list[str]: + rows = ( + ( + await session.execute( + select(MetricDataQuality.detail) + .where(MetricDataQuality.check_name.in_(INCOME_CHECKS)) + .order_by(MetricDataQuality.check_name) + ) + ) + .scalars() + .all() + ) + return list(rows) diff --git a/backend/src/fintracker/api/routers/rebalance.py b/backend/src/fintracker/api/routers/rebalance.py new file mode 100644 index 0000000..bbf0781 --- /dev/null +++ b/backend/src/fintracker/api/routers/rebalance.py @@ -0,0 +1,231 @@ +"""Target weights and rebalancing suggestions (docs/ai/phase4-contract.md §2). + +Unlike the read-only `/analytics/*` endpoints, the suggestion here is computed per request +through `analytics/rebalance.compute_rebalance` rather than read from `metric_rebalance`. +Two reasons, and they are the same reason twice: the what-if parameter `cash_available` +changes the answer and therefore cannot come from a stored table, and a target the user just +saved must be visible before the next refresh — a screen that shows yesterday's targets +next to today's prices is a screen that lies. The stored table stays the canonical copy for +everything that reads metrics in bulk, and both come out of the same function, so they +cannot disagree. +""" + +from __future__ import annotations + +from decimal import Decimal +from typing import Annotated + +from fastapi import APIRouter, Query, status +from sqlalchemy import delete, select + +from fintracker.analytics.rebalance import DEFAULT_DIMENSION, compute_rebalance +from fintracker.api.deps import CurrentUser, SessionDep +from fintracker.api.errors import Problem +from fintracker.api.schemas.rebalance import ( + RebalanceBucketOut, + RebalanceOut, + TargetOut, + TargetsIn, + TargetsOut, + TradeOut, +) +from fintracker.models import AllocationDimension, Portfolio, PortfolioTarget + +router = APIRouter(prefix="/portfolios", tags=["rebalance"]) + +ZERO = Decimal(0) +ONE = Decimal(1) + +WEIGHT_TOLERANCE = Decimal("0.0001") +"""How far the weights of one dimension may add up from 1 before the set is refused. +The server does not normalise: a total of 0.9 is a mistake in the plan, not a scale factor, +and silently stretching it would hide the missing tenth of the portfolio forever.""" + +DimensionParam = Annotated[str, Query(description="asset_class | sector | country | currency")] + + +def parse_dimension(value: str) -> AllocationDimension: + try: + return AllocationDimension(value) + except ValueError: + allowed = ", ".join(d.value for d in AllocationDimension) + raise Problem( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "Validation error", + f"Неизвестное измерение: {value}. Допустимые: {allowed}", + ) from None + + +async def _portfolio(session: SessionDep, portfolio_id: int) -> Portfolio: + portfolio = await session.get(Portfolio, portfolio_id) + if portfolio is None: + raise Problem(status.HTTP_404_NOT_FOUND, "Not found", f"Нет портфеля с id {portfolio_id}") + return portfolio + + +async def _targets_out( + session: SessionDep, portfolio_id: int, dimension: AllocationDimension +) -> TargetsOut: + rows = ( + ( + await session.execute( + select(PortfolioTarget) + .where( + PortfolioTarget.portfolio_id == portfolio_id, + PortfolioTarget.dimension == dimension, + ) + .order_by(PortfolioTarget.bucket) + ) + ) + .scalars() + .all() + ) + return TargetsOut( + portfolio_id=portfolio_id, + dimension=dimension.value, + targets=[ + TargetOut( + bucket=r.bucket, + target_weight=Decimal(r.target_weight), + band=None if r.band is None else Decimal(r.band), + note=r.note, + ) + for r in rows + ], + weights_sum=sum((Decimal(r.target_weight) for r in rows), start=ZERO), + ) + + +@router.get("/{portfolio_id}/targets", name="targets") +async def get_targets( + portfolio_id: int, + session: SessionDep, + _: CurrentUser, + dimension: DimensionParam = DEFAULT_DIMENSION.value, +) -> TargetsOut: + await _portfolio(session, portfolio_id) + return await _targets_out(session, portfolio_id, parse_dimension(dimension)) + + +@router.put("/{portfolio_id}/targets", name="set_targets") +async def put_targets( + portfolio_id: int, body: TargetsIn, session: SessionDep, _: CurrentUser +) -> TargetsOut: + """Replace the whole set for one dimension. + + The weights must add up to 1 within `WEIGHT_TOLERANCE`; otherwise the request is + refused with the actual sum in the message, so the user can see by how much the plan + misses rather than being handed a silently rescaled one. + """ + await _portfolio(session, portfolio_id) + dimension = parse_dimension(body.dimension) + + buckets = [t.bucket for t in body.targets] + duplicates = sorted({b for b in buckets if buckets.count(b) > 1}) + if duplicates: + raise Problem( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "Validation error", + f"Бакет указан дважды: {', '.join(duplicates)}", + ) + for target in body.targets: + if target.target_weight < ZERO or target.target_weight > ONE: + raise Problem( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "Validation error", + f"Вес бакета «{target.bucket}» вне диапазона 0..1: {target.target_weight}", + ) + if target.band is not None and target.band < ZERO: + raise Problem( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "Validation error", + f"Полоса допуска бакета «{target.bucket}» отрицательна: {target.band}", + ) + + total = sum((t.target_weight for t in body.targets), start=ZERO) + if body.targets and abs(total - ONE) > WEIGHT_TOLERANCE: + raise Problem( + status.HTTP_422_UNPROCESSABLE_ENTITY, + "Validation error", + f"Сумма весов должна быть 1, а не {format(total, 'f')}. " + f"Сервер не нормализует веса — исправьте набор.", + extra={"weights_sum": format(total, "f")}, + ) + + await session.execute( + delete(PortfolioTarget).where( + PortfolioTarget.portfolio_id == portfolio_id, + PortfolioTarget.dimension == dimension, + ) + ) + for target in body.targets: + session.add( + PortfolioTarget( + portfolio_id=portfolio_id, + dimension=dimension, + bucket=target.bucket, + target_weight=target.target_weight, + band=target.band, + note=target.note, + ) + ) + await session.commit() + return await _targets_out(session, portfolio_id, dimension) + + +@router.get("/{portfolio_id}/rebalance", name="rebalance") +async def get_rebalance( + portfolio_id: int, + session: SessionDep, + _: CurrentUser, + dimension: DimensionParam = DEFAULT_DIMENSION.value, + cash_available: Annotated[ + Decimal | None, Query(description="переопределяет остаток на счетах для what-if") + ] = None, +) -> RebalanceOut: + await _portfolio(session, portfolio_id) + plan = await compute_rebalance( + session, + portfolio_id, + parse_dimension(dimension), + cash_available_rub=cash_available, + ) + return RebalanceOut( + portfolio_id=plan.portfolio_id, + dimension=plan.dimension.value, + as_of=plan.as_of, + total_value_rub=plan.total_value_rub, + cash_available_rub=plan.cash_available_rub, + buckets=[ + RebalanceBucketOut( + bucket=b.bucket, + current_value_rub=b.current_value_rub, + current_weight=b.current_weight, + target_weight=b.target_weight, + drift=b.drift, + within_band=b.within_band, + delta_value_rub=b.delta_value_rub, + trades=[ + TradeOut( + instrument_id=t.instrument_id, + ticker=t.ticker, + name=t.name, + action=t.action, + # inside the band nothing is proposed, so the quantity is NULL + suggested_qty=None if b.within_band else t.qty, + lot=t.lot, + price=t.price, + price_currency=t.price_currency, + amount_rub=t.amount_rub, + blocked_by_cash=t.blocked_by_cash, + ) + for t in b.trades + ], + ) + for b in plan.buckets + ], + warnings=plan.warnings, + ) + + +__all__ = ["router"] diff --git a/backend/src/fintracker/api/routers/tax.py b/backend/src/fintracker/api/routers/tax.py new file mode 100644 index 0000000..4dad3bd --- /dev/null +++ b/backend/src/fintracker/api/routers/tax.py @@ -0,0 +1,116 @@ +"""Tax: the year's summary, and the open lots with their ЛДВ dates. + +`GET /tax` reads `metric_tax_year`, which the refresh built. `GET /tax/lots` is computed per +request on purpose — it depends on today's date and today's price, and a nightly table would +answer `days_to_ldv` as of last night, which is off by one on precisely the day the number +is being consulted. + +Both responses carry `estimated: true`, the rate that produced them and a disclaimer. The +broker is the tax agent; see `analytics/tax.py` for what is and is not modelled. +""" + +from __future__ import annotations + +from dataclasses import asdict +from decimal import Decimal +from typing import Annotated + +from fastapi import APIRouter, Query +from sqlalchemy import select + +from fintracker.analytics import today_local +from fintracker.analytics.tax import DISCLAIMER, TAX_RATE, money, open_lot_tax +from fintracker.api.deps import CurrentUser, SessionDep +from fintracker.api.schemas.tax import ( + TaxAccountOut, + TaxLotOut, + TaxLotsOut, + TaxTotals, + TaxYearOut, +) +from fintracker.models import Account, MetricTaxYear + +router = APIRouter(prefix="/tax", tags=["tax"]) + +ZERO = Decimal(0) + +TOTAL_FIELDS = ( + "dividends_gross_rub", + "coupons_gross_rub", + "tax_withheld_rub", + "realized_gain_rub", + "realized_loss_rub", + "ldv_exempt_rub", + "taxable_base_rub", + "estimated_tax_rub", +) + + +@router.get("", name="summary") +async def tax_year( + session: SessionDep, + _: CurrentUser, + year: Annotated[int | None, Query(ge=1990, le=2200)] = None, + account_id: int | None = None, +) -> TaxYearOut: + """The year's estimated tax position, per account and in total.""" + year = year or today_local().year + conditions = [MetricTaxYear.year == year] + if account_id is not None: + conditions.append(MetricTaxYear.account_id == account_id) + rows = ( + await session.execute( + select(MetricTaxYear, Account.name) + .join(Account, Account.id == MetricTaxYear.account_id) + .where(*conditions) + .order_by(Account.name) + ) + ).all() + + accounts = [ + TaxAccountOut( + account_id=r[0].account_id, + account_name=r[1], + **{f: getattr(r[0], f) for f in TOTAL_FIELDS}, + ) + for r in rows + ] + # the rate is read back from the rows rather than assumed, so a year computed under an + # older rate keeps reporting the rate it was computed with + rate = rows[0][0].tax_rate if rows else TAX_RATE + return TaxYearOut( + year=year, + tax_rate=rate, + accounts=accounts, + totals=TaxTotals( + **{f: money(sum((getattr(a, f) for a in accounts), start=ZERO)) for f in TOTAL_FIELDS} + ), + disclaimer=DISCLAIMER, + ) + + +@router.get("/lots", name="lots") +async def tax_lots( + session: SessionDep, + _: CurrentUser, + year: Annotated[int | None, Query(ge=1990, le=2200)] = None, + account_id: int | None = None, +) -> TaxLotsOut: + """Open lots with the date after which a sale falls under the long-term exemption. + + `year` selects the rate the "what if I sold today" figure is computed at; the lots + themselves are always the ones open right now. `lot` is rebuilt from the ledger on every + refresh and holds today's state only, so there is no honest way to answer "which lots + were open on 31 December two years ago" — and inventing one would be worse than the + limitation. + """ + as_of = today_local() + year = year or as_of.year + lots = await open_lot_tax(session, as_of=as_of, account_id=account_id) + return TaxLotsOut( + year=year, + as_of=as_of, + tax_rate=TAX_RATE, + lots=[TaxLotOut(**asdict(lot)) for lot in lots], + disclaimer=DISCLAIMER, + ) diff --git a/backend/src/fintracker/api/schemas/benchmarks.py b/backend/src/fintracker/api/schemas/benchmarks.py new file mode 100644 index 0000000..b9c3955 --- /dev/null +++ b/backend/src/fintracker/api/schemas/benchmarks.py @@ -0,0 +1,87 @@ +"""Schemas for the benchmark list and the portfolio-vs-index comparison (phase-4 contract §3). + +`kind` crosses the wire as a plain string (`price` | `total_return`), like every other stable +key in this API — and unlike a Dart enum, which the generated client cannot always name. The +client is expected to *use* it: a portfolio compared against a price index is being compared +against a series that throws its dividends away, and the screen has to say so. +""" + +from __future__ import annotations + +from datetime import date + +from pydantic import BaseModel, Field + +from fintracker.api.schemas.common import MoneyOpt + + +class BenchmarkOut(BaseModel): + id: int + code: str + """MOEX secid or another stable code: IMOEX, MCFTR, RGBITR.""" + name: str + kind: str + """price | total_return — a price index understates a holder's result by its dividends.""" + source: str + """moex | manual""" + currency: str + is_default: bool + is_active: bool + instrument_id: int | None + """The instrument carrying the index history; null until the index has been synced.""" + history_from: date | None + history_to: date | None + """Range actually present in `price_daily`; null when there is no history at all.""" + + +class BenchmarkCreate(BaseModel): + code: str = Field(min_length=1, max_length=32) + name: str = Field(min_length=1, max_length=128) + kind: str + """price | total_return""" + source: str = "moex" + currency: str = "RUB" + is_default: bool = False + is_active: bool = True + instrument_id: int | None = None + + +class BenchmarkPatch(BaseModel): + code: str | None = Field(default=None, min_length=1, max_length=32) + name: str | None = Field(default=None, min_length=1, max_length=128) + kind: str | None = None + source: str | None = None + currency: str | None = None + is_default: bool | None = None + is_active: bool | None = None + instrument_id: int | None = None + + +class BenchmarkReturnOut(BaseModel): + benchmark_id: int + code: str + name: str + kind: str + twr: MoneyOpt + """Cumulative over the period; null when the index had no quote to start from.""" + twr_annualized: MoneyOpt + days_skipped: int + """Days of the compared window the index had no quote for. Non-zero means the two series + are not day-for-day comparable, and the client must show it.""" + excess: MoneyOpt + """portfolio_twr - twr; null when either side is unknown.""" + + +class BenchmarkComparisonOut(BaseModel): + period: str + """1m | 3m | 6m | ytd | 1y | 3y | all""" + date_from: date + date_to: date + portfolio_twr: MoneyOpt + portfolio_twr_annualized: MoneyOpt + portfolio_days_skipped: int + benchmarks: list[BenchmarkReturnOut] + + +class BenchmarkComparison(BaseModel): + rows: list[BenchmarkComparisonOut] diff --git a/backend/src/fintracker/api/schemas/goals.py b/backend/src/fintracker/api/schemas/goals.py new file mode 100644 index 0000000..36cc2cc --- /dev/null +++ b/backend/src/fintracker/api/schemas/goals.py @@ -0,0 +1,63 @@ +"""Wire shapes for goals and their progress (docs/ai/phase4-contract.md §4).""" + +from __future__ import annotations + +from datetime import date + +from pydantic import BaseModel, ConfigDict, Field + +from fintracker.api.schemas.common import Money, MoneyOpt + + +class GoalOut(BaseModel): + id: int + name: str + scope: str + target_amount: Money + currency: str + target_date: date | None + monthly_contribution: MoneyOpt + note: str | None + archived: bool + + +class GoalCreate(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str = Field(min_length=1, max_length=128) + scope: str = Field(default="all", max_length=32) + target_amount: Money + currency: str = Field(default="RUB", min_length=3, max_length=3) + target_date: date | None = None + monthly_contribution: MoneyOpt = None + note: str | None = None + archived: bool = False + + +class GoalPatch(BaseModel): + model_config = ConfigDict(extra="forbid") + + name: str | None = Field(default=None, min_length=1, max_length=128) + scope: str | None = Field(default=None, max_length=32) + target_amount: MoneyOpt = None + currency: str | None = Field(default=None, min_length=3, max_length=3) + target_date: date | None = None + monthly_contribution: MoneyOpt = None + note: str | None = None + archived: bool | None = None + + +class GoalProgressOut(BaseModel): + goal_id: int + as_of: date + current_value_rub: Money + target_amount_rub: Money + progress: Money + projected_date: date | None + """NULL means the current trend does not reach the target — never a far-future date.""" + basis: str + """xirr | contribution | none""" + assumed_rate: MoneyOpt + monthly_needed_rub: MoneyOpt + """NULL when the goal has no deadline, or the deadline has already passed.""" + on_track: bool | None diff --git a/backend/src/fintracker/api/schemas/income.py b/backend/src/fintracker/api/schemas/income.py new file mode 100644 index 0000000..82dafe8 --- /dev/null +++ b/backend/src/fintracker/api/schemas/income.py @@ -0,0 +1,84 @@ +"""Schemas for `/income` — the dividend and coupon calendar (phase-4 contract §1). + +`basis` is a plain string here and on every row, never an enum and never optional: a calendar +that shows «12 400 ₽ ожидается» without saying whether that is a published coupon schedule or +an extrapolation from two payments is not a forecast the user can audit. The client is +required to render it. + +`amount_rub` is nullable for the usual reason — the payment's date had no CBR rate — and the +client shows that as «нет курса», not as 0 ₽. +""" + +from __future__ import annotations + +from datetime import date + +from pydantic import BaseModel + +from fintracker.api.schemas.common import Money, MoneyOpt + + +class CalendarEntry(BaseModel): + instrument_id: int + ticker: str | None + name: str + kind: str + """dividend | coupon | amortization | repayment""" + expected_date: date + record_date: date | None + qty: Money + """Position the amount was computed on; 0 on a historical row, which carries its own sum.""" + per_unit: MoneyOpt + amount: Money + currency: str + amount_rub: MoneyOpt + """Null when the date has no rate — never a substituted number.""" + basis: str + """schedule | announced | history | paid — where the number came from.""" + tax_withheld: MoneyOpt + + +class CalendarOut(BaseModel): + as_of: date + currency: str + total_expected_rub: Money + """Sum of the FUTURE entries only; paid rows never inflate an expectation.""" + entries: list[CalendarEntry] + by_basis: dict[str, Money] + """Expected RUB split by basis. Sums that mix `schedule` with `history` hide the guess.""" + + +class HistoryRow(BaseModel): + month: date + """First day of the month.""" + kind: str + currency: str + amount: Money + amount_rub: MoneyOpt + tax_withheld: Money + payment_count: int + + +class HistoryTotals(BaseModel): + amount_rub: Money + tax_withheld_rub: Money + + +class HistoryOut(BaseModel): + rows: list[HistoryRow] + totals: HistoryTotals + + +class ForecastMonth(BaseModel): + month: date + amount_rub: Money + by_basis: dict[str, Money] + + +class ForecastOut(BaseModel): + months: list[ForecastMonth] + total_rub: Money + annual_yield_on_value: MoneyOpt + """Expected 12-month income over the scope's current value; null without a valuation.""" + warnings: list[str] + """What the forecast could not cover, verbatim from the data-quality findings.""" diff --git a/backend/src/fintracker/api/schemas/rebalance.py b/backend/src/fintracker/api/schemas/rebalance.py new file mode 100644 index 0000000..5da0fec --- /dev/null +++ b/backend/src/fintracker/api/schemas/rebalance.py @@ -0,0 +1,87 @@ +"""Wire shapes for target weights and rebalancing (docs/ai/phase4-contract.md §2). + +`dimension`, `bucket` and `action` are plain strings, never enums: `AssetClass` must not +leak into the generated Dart client (AGENTS.md), and the same rule keeps every stable key +a string. Weights, drifts and quantities are Decimals serialised as strings — a weight is +`"0.6"`, not `60`. +""" + +from __future__ import annotations + +from datetime import date + +from pydantic import BaseModel, ConfigDict, Field + +from fintracker.api.schemas.common import Money, MoneyOpt + + +class TargetIn(BaseModel): + model_config = ConfigDict(extra="forbid") + + bucket: str = Field(min_length=1, max_length=64) + target_weight: Money + band: MoneyOpt = None + """Tolerance in the same units as the weight: `"0.05"` is ±5 pp, not 5 %.""" + note: str | None = None + + +class TargetsIn(BaseModel): + """A complete set for one dimension. Partial updates are not supported: weights only + mean anything together, and a half-written set could not be checked against 1.""" + + model_config = ConfigDict(extra="forbid") + + dimension: str = Field(description="asset_class | sector | country | currency") + targets: list[TargetIn] + + +class TargetOut(BaseModel): + bucket: str + target_weight: Money + band: MoneyOpt + note: str | None + + +class TargetsOut(BaseModel): + portfolio_id: int + dimension: str + targets: list[TargetOut] + weights_sum: Money + """What the weights actually add up to. The server never normalises them.""" + + +class TradeOut(BaseModel): + instrument_id: int + ticker: str + name: str + action: str + """buy | sell""" + suggested_qty: MoneyOpt + """Whole lots; NULL when the instrument has no usable price.""" + lot: int | None + price: MoneyOpt + price_currency: str | None + amount_rub: Money + blocked_by_cash: bool + + +class RebalanceBucketOut(BaseModel): + bucket: str + current_value_rub: Money + current_weight: Money + target_weight: MoneyOpt + drift: MoneyOpt + """current - target, in fractions of the whole portfolio.""" + within_band: bool + delta_value_rub: Money + trades: list[TradeOut] + + +class RebalanceOut(BaseModel): + portfolio_id: int + dimension: str + as_of: date + total_value_rub: Money + cash_available_rub: Money + buckets: list[RebalanceBucketOut] + warnings: list[str] diff --git a/backend/src/fintracker/api/schemas/tax.py b/backend/src/fintracker/api/schemas/tax.py new file mode 100644 index 0000000..8224069 --- /dev/null +++ b/backend/src/fintracker/api/schemas/tax.py @@ -0,0 +1,86 @@ +"""Schemas for the tax screens (phase-4 contract §5). + +Every response carries `estimated: true`, the `tax_rate` that produced it and a `disclaimer`, +and none of the three is optional. The broker is the tax agent; these numbers exist so its +certificate can be checked and so the cost of selling before the three-year mark is visible +*before* the sale, not to replace it. A client that drops the marking is showing a number it +is not entitled to show. +""" + +from __future__ import annotations + +from datetime import date + +from pydantic import BaseModel + +from fintracker.api.schemas.common import Money, MoneyOpt + + +class TaxTotals(BaseModel): + dividends_gross_rub: Money + coupons_gross_rub: Money + tax_withheld_rub: Money + """What the broker already held back on those payments.""" + realized_gain_rub: Money + """Sum of the profitable disposals of the year, LDV ones included.""" + realized_loss_rub: Money + """Sum of the losing ones, negative.""" + ldv_exempt_rub: Money + """Net result of the disposals exempt under art. 219.1, taken back out of the base. + Negative if those lots lost money: the exemption removes the whole result, not the + profitable half.""" + taxable_base_rub: Money + """max(0, gain + loss - ldv_exempt). Dividends and coupons are not in it — the agent + withholds on them at source.""" + estimated_tax_rub: Money + + +class TaxAccountOut(TaxTotals): + account_id: int + account_name: str + + +class TaxYearOut(BaseModel): + year: int + estimated: bool = True + """Always true. There is no mode in which this endpoint returns an authoritative figure.""" + tax_rate: Money + """The rate actually applied, as a fraction: "0.13", never 13.""" + accounts: list[TaxAccountOut] + totals: TaxTotals + disclaimer: str + + +class TaxLotOut(BaseModel): + lot_id: int + instrument_id: int + ticker: str | None + name: str + account_id: int + account_name: str + open_date: date + qty_remaining: Money + cost_rub: MoneyOpt + """Cost at the CBR rate of the lot's own open date, prorated to the part still held; + null when that day had no rate.""" + market_value_rub: MoneyOpt + """Null when the instrument has no price — never zero.""" + unrealized_gain_rub: MoneyOpt + ldv_eligible: bool + """Held three full years on an exchange-traded instrument (art. 219.1) — an estimate: + the classic rule only, MOEX papers only, ИИС regimes not modelled.""" + ldv_date: date | None + """First day a sale qualifies; null for an instrument that cannot qualify at all.""" + days_to_ldv: int | None + """Calendar days left, 0 once eligible; null alongside a null `ldv_date`.""" + tax_if_sold_now_rub: MoneyOpt + """0 for an eligible lot; null when the position cannot be valued.""" + + +class TaxLotsOut(BaseModel): + year: int + as_of: date + estimated: bool = True + tax_rate: Money + lots: list[TaxLotOut] + disclaimer: str diff --git a/backend/src/fintracker/models/__init__.py b/backend/src/fintracker/models/__init__.py index e5c8ef5..4ee695a 100644 --- a/backend/src/fintracker/models/__init__.py +++ b/backend/src/fintracker/models/__init__.py @@ -25,19 +25,33 @@ from fintracker.models.ledger import ( ) from fintracker.models.metrics import ( AllocationDimension, + IncomeBasis, MetricAllocation, + MetricBenchmarkReturns, MetricCashFlowBroker, MetricCashFlowMonthly, MetricDataQuality, + MetricGoalProgress, MetricHolding, + MetricIncomeCalendar, + MetricIncomeMonthly, MetricNetWorthDaily, MetricPortfolioValueDaily, + MetricRebalance, MetricRefreshLog, MetricReturns, MetricRunway, MetricSpendingByCategory, + MetricTaxYear, +) +from fintracker.models.planning import ( + Benchmark, + BenchmarkKind, + Goal, + PortfolioTarget, ) from fintracker.models.pricing import ( + BondNominalSchedule, CashSnapshot, CorporateAction, CorporateActionKind, @@ -95,6 +109,9 @@ __all__ = [ "AllocationDimension", "AppUser", "AssetClass", + "Benchmark", + "BenchmarkKind", + "BondNominalSchedule", "Broker", "CashSnapshot", "CashTxn", @@ -111,6 +128,8 @@ __all__ = [ "FlowLinkKind", "FlowType", "FxRateDaily", + "Goal", + "IncomeBasis", "Instrument", "InstrumentAlias", "JobStatus", @@ -118,20 +137,27 @@ __all__ = [ "LotDisposal", "Merchant", "MetricAllocation", + "MetricBenchmarkReturns", "MetricCashFlowBroker", "MetricCashFlowMonthly", "MetricDataQuality", + "MetricGoalProgress", "MetricHolding", + "MetricIncomeCalendar", + "MetricIncomeMonthly", "MetricNetWorthDaily", "MetricPortfolioValueDaily", + "MetricRebalance", "MetricRefreshLog", "MetricReturns", "MetricRunway", "MetricSpendingByCategory", + "MetricTaxYear", "PendingInstrument", "PendingInstrumentStatus", "Portfolio", "PortfolioAccount", + "PortfolioTarget", "PositionSnapshot", "PriceCoverage", "PriceDaily", diff --git a/backend/src/fintracker/models/metrics.py b/backend/src/fintracker/models/metrics.py index 00100cb..2dc5fc3 100644 --- a/backend/src/fintracker/models/metrics.py +++ b/backend/src/fintracker/models/metrics.py @@ -8,7 +8,7 @@ from datetime import date, datetime from decimal import Decimal from typing import Any -from sqlalchemy import ForeignKey, Integer, String, Text, UniqueConstraint, func +from sqlalchemy import Boolean, ForeignKey, Integer, String, Text, UniqueConstraint, func from sqlalchemy.orm import Mapped, mapped_column from fintracker.db.base import Base, db_enum @@ -284,4 +284,195 @@ class MetricAllocation(Base): """Share of the scope's valued total; the weights of one dimension add up to 1.""" holding_count: Mapped[int] = mapped_column(Integer, default=0) """Instruments in this bucket; 0 for the cash bucket.""" + target_weight: Mapped[Decimal | None] + """From `portfolio_target`; NULL when the user set no target for this bucket.""" + drift: Mapped[Decimal | None] + """`weight - target_weight` in percentage points of the whole, NULL without a target.""" + computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + +class IncomeBasis(enum.StrEnum): + """Where a calendar entry's number comes from — the honesty column. + + A bond's future coupon is arithmetic on a published schedule; an announced dividend is a + fact the issuer stated; a projected dividend is an extrapolation from history and can be + wrong by any amount. Mixing the three into one «expected income» figure is the fastest + way to make a forecast that nobody can audit. + """ + + schedule = "schedule" + """Bond coupon from the payment schedule and the nominal in force on that date.""" + announced = "announced" + """Declared by the issuer: a `corporate_action` with a future record date.""" + history = "history" + """Extrapolated from the last 24 months — a guess, labelled as one.""" + paid = "paid" + """Already received; the entry is history, not a forecast.""" + + +class MetricIncomeCalendar(Base): + """Dividends and coupons, past and expected, per scope and instrument (plan §3). + + Rows are per payment, not per month, because the screen the user actually wants answers + «what lands, and when» — and because an amount that turns out wrong has to be traceable + to the single assumption that produced it (`basis`, `per_unit`, `qty`). + """ + + __tablename__ = "metric_income_calendar" + __table_args__ = (UniqueConstraint("scope", "instrument_id", "kind", "expected_date", "basis"),) + + id: Mapped[int] = mapped_column(primary_key=True) + scope: Mapped[str] = mapped_column(String(32), index=True) + instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE")) + kind: Mapped[str] = mapped_column(String(16)) + """dividend | coupon | amortization | repayment — the `corporate_action` kinds that pay.""" + expected_date: Mapped[date] = mapped_column(index=True) + record_date: Mapped[date | None] + qty: Mapped[Decimal] + """Position held when the payment is expected; 0 once the position is gone.""" + per_unit: Mapped[Decimal | None] + amount: Mapped[Decimal] + currency: Mapped[str] = mapped_column(String(3)) + amount_rub: Mapped[Decimal | None] + """NULL when the date has no FX rate — never a substituted number.""" + basis: Mapped[IncomeBasis] = mapped_column(db_enum(IncomeBasis, "income_basis")) + tax_withheld: Mapped[Decimal | None] + """What the broker already held back, for payments that happened.""" + computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + +class MetricIncomeMonthly(Base): + """Income actually received, per scope, month and kind — the history half of the screen.""" + + __tablename__ = "metric_income_monthly" + __table_args__ = (UniqueConstraint("scope", "month", "kind", "currency"),) + + id: Mapped[int] = mapped_column(primary_key=True) + scope: Mapped[str] = mapped_column(String(32), index=True) + month: Mapped[date] + """First day of the month.""" + kind: Mapped[str] = mapped_column(String(16)) + currency: Mapped[str] = mapped_column(String(3)) + amount: Mapped[Decimal] + amount_rub: Mapped[Decimal | None] + tax_withheld: Mapped[Decimal] + payment_count: Mapped[int] = mapped_column(Integer, default=0) + computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + +class MetricBenchmarkReturns(Base): + """A benchmark's TWR over the same period and date grid as `metric_returns`. + + Kept beside the portfolio's own returns rather than inside them: the set of benchmarks is + a user's choice, and a column per index would mean a migration per index. Comparing + requires the same grid — a benchmark measured over a different set of days is not a + comparison, so `date_from`/`date_to` are stored and checked. + """ + + __tablename__ = "metric_benchmark_returns" + __table_args__ = (UniqueConstraint("scope", "period", "benchmark_id"),) + + id: Mapped[int] = mapped_column(primary_key=True) + scope: Mapped[str] = mapped_column(String(32), index=True) + period: Mapped[str] = mapped_column(String(8)) + benchmark_id: Mapped[int] = mapped_column(ForeignKey("benchmark.id", ondelete="CASCADE")) + date_from: Mapped[date] + date_to: Mapped[date] + twr: Mapped[Decimal | None] + twr_annualized: Mapped[Decimal | None] + days_skipped: Mapped[int] = mapped_column(Integer, default=0) + """Days the index had no quote for. Non-zero means the comparison is not like-for-like.""" + computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + +class MetricRebalance(Base): + """What to buy or sell to reach the target weights (plan §3). + + `suggested_qty` respects the lot size and the cash on hand, because a recommendation that + cannot be executed is not a recommendation. It is whole lots, never a fractional share. + """ + + __tablename__ = "metric_rebalance" + __table_args__ = (UniqueConstraint("portfolio_id", "dimension", "bucket", "instrument_id"),) + + id: Mapped[int] = mapped_column(primary_key=True) + portfolio_id: Mapped[int] = mapped_column(ForeignKey("portfolio.id", ondelete="CASCADE")) + dimension: Mapped[AllocationDimension] = mapped_column( + db_enum(AllocationDimension, "allocation_dimension") + ) + bucket: Mapped[str] = mapped_column(String(64)) + instrument_id: Mapped[int | None] = mapped_column( + ForeignKey("instrument.id", ondelete="CASCADE") + ) + """NULL on the bucket's own summary row; set on each proposed trade inside it.""" + current_value_rub: Mapped[Decimal] + current_weight: Mapped[Decimal] + target_weight: Mapped[Decimal | None] + delta_value_rub: Mapped[Decimal] + """Positive: buy this much. Negative: sell.""" + suggested_qty: Mapped[Decimal | None] + """Whole lots; NULL when the instrument has no usable price.""" + lot: Mapped[int | None] = mapped_column(Integer) + price: Mapped[Decimal | None] + price_currency: Mapped[str | None] = mapped_column(String(3)) + within_band: Mapped[bool] = mapped_column(Boolean, default=False) + """True when the drift is inside `portfolio_target.band` — shown, but not acted on.""" + blocked_by_cash: Mapped[bool] = mapped_column(Boolean, default=False) + """A buy the available cash does not cover; the quantity is what the cash does allow.""" + computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + +class MetricTaxYear(Base): + """The tax picture per calendar year and account (plan §3, ст. 214.1 / 219.1 НК). + + Estimated, and labelled estimated everywhere it surfaces: the broker is the tax agent and + the authoritative number is its own certificate. This exists to make that certificate + checkable and to show the cost of selling a position before the three-year mark. + """ + + __tablename__ = "metric_tax_year" + __table_args__ = (UniqueConstraint("year", "account_id"),) + + id: Mapped[int] = mapped_column(primary_key=True) + year: Mapped[int] = mapped_column(Integer, index=True) + account_id: Mapped[int] = mapped_column(ForeignKey("account.id", ondelete="CASCADE")) + dividends_gross_rub: Mapped[Decimal] + coupons_gross_rub: Mapped[Decimal] + tax_withheld_rub: Mapped[Decimal] + """What the broker already took — the difference between gross and what arrived.""" + realized_gain_rub: Mapped[Decimal] + """FIFO result in roubles, each leg at the CBR rate of its own date (currency revaluation + is part of the base, plan §7 вопрос 4).""" + realized_loss_rub: Mapped[Decimal] + ldv_exempt_rub: Mapped[Decimal] + """Gain on lots held 3+ years, exempt under art. 219.1 — an estimate, see the docstring.""" + taxable_base_rub: Mapped[Decimal] + estimated_tax_rub: Mapped[Decimal] + tax_rate: Mapped[Decimal] + """The rate applied, so a changed rate is visible rather than baked into the number.""" + computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) + + +class MetricGoalProgress(Base): + """Where a goal stands and when it is projected to be met.""" + + __tablename__ = "metric_goal_progress" + + goal_id: Mapped[int] = mapped_column( + ForeignKey("goal.id", ondelete="CASCADE"), primary_key=True + ) + as_of: Mapped[date] + current_value_rub: Mapped[Decimal] + target_amount_rub: Mapped[Decimal] + progress: Mapped[Decimal] + """current / target, clamped to [0, …]; can exceed 1 for a goal already met.""" + projected_date: Mapped[date | None] + """NULL when the trend does not reach the target (flat or falling), never a far-future + date pretending to be an answer.""" + basis: Mapped[str] = mapped_column(String(16)) + """xirr | contribution | none — which assumption produced `projected_date`.""" + assumed_rate: Mapped[Decimal | None] + monthly_needed_rub: Mapped[Decimal | None] + """Contribution required to hit `target_date`; NULL when the goal has no deadline.""" + on_track: Mapped[bool | None] computed_at: Mapped[datetime] = mapped_column(server_default=func.now()) diff --git a/backend/src/fintracker/models/planning.py b/backend/src/fintracker/models/planning.py new file mode 100644 index 0000000..7ccea1a --- /dev/null +++ b/backend/src/fintracker/models/planning.py @@ -0,0 +1,106 @@ +"""What the portfolio is *supposed* to look like: targets, benchmarks, goals (plan §4, фаза 4). + +Everything else in `models/` records what happened. These three record intent, and that is +why they are separate: a target weight is not a measurement, it is a decision the user made, +and a refresh that rebuilds every metric from scratch must never touch it. + +* `PortfolioTarget` — the desired split along one allocation dimension. Rebalancing compares + it against `metric_allocation` and proposes trades. +* `Benchmark` — an index the portfolio is measured against. Stored as a row rather than a + constant because the comparison list is a user's choice, and because an index quoted on + MOEX is an ordinary `instrument` whose history already lands in `price_daily`. +* `Goal` — an amount by a date. Progress is derived, never stored here. +""" + +from __future__ import annotations + +import enum +from datetime import date +from decimal import Decimal + +from sqlalchemy import Boolean, ForeignKey, String, Text, UniqueConstraint +from sqlalchemy.orm import Mapped, mapped_column + +from fintracker.db.base import Base, TimestampMixin, db_enum +from fintracker.models.metrics import AllocationDimension + + +class PortfolioTarget(TimestampMixin, Base): + """Target weight of one bucket of one dimension, as a fraction (0.25 = 25 %). + + Keyed by the same `(dimension, bucket)` pair `metric_allocation` uses, so the two join + directly and a bucket that exists in only one of them is visible rather than silently + dropped — an allocation that drifted to zero still owes an answer. + """ + + __tablename__ = "portfolio_target" + __table_args__ = (UniqueConstraint("portfolio_id", "dimension", "bucket"),) + + id: Mapped[int] = mapped_column(primary_key=True) + portfolio_id: Mapped[int] = mapped_column( + ForeignKey("portfolio.id", ondelete="CASCADE"), index=True + ) + dimension: Mapped[AllocationDimension] = mapped_column( + db_enum(AllocationDimension, "allocation_dimension") + ) + bucket: Mapped[str] = mapped_column(String(64)) + target_weight: Mapped[Decimal] + """Fraction of the portfolio. Weights of one dimension should add to 1; the API reports + the shortfall rather than normalising, because a total of 0.9 is a mistake, not a scale.""" + band: Mapped[Decimal | None] + """Tolerance around the target below which no rebalancing is suggested (0.05 = ±5 pp). + Without it every price tick produces a trade recommendation.""" + note: Mapped[str | None] = mapped_column(Text) + + +class BenchmarkKind(enum.StrEnum): + price = "price" + """Price index — no dividends (IMOEX). Understates what a holder actually earned.""" + total_return = "total_return" + """Gross total-return index — dividends reinvested (MCFTR). The honest comparison.""" + + +class Benchmark(TimestampMixin, Base): + """An index the portfolio's TWR is compared against.""" + + __tablename__ = "benchmark" + + id: Mapped[int] = mapped_column(primary_key=True) + code: Mapped[str] = mapped_column(String(32), unique=True) + """MOEX secid or another stable code: IMOEX, MCFTR, RGBITR.""" + name: Mapped[str] = mapped_column(String(128)) + kind: Mapped[BenchmarkKind] = mapped_column(db_enum(BenchmarkKind, "benchmark_kind")) + instrument_id: Mapped[int | None] = mapped_column( + ForeignKey("instrument.id", ondelete="SET NULL") + ) + """The `instrument` carrying the index's price history; NULL until it is first synced.""" + source: Mapped[str] = mapped_column(String(16), default="moex") + """moex | manual — the S&P 500 has no free official feed (plan §7, вопрос 2).""" + currency: Mapped[str] = mapped_column(String(3), default="RUB") + is_default: Mapped[bool] = mapped_column(Boolean, default=False) + """Shown next to the portfolio's own return without being asked for.""" + is_active: Mapped[bool] = mapped_column(Boolean, default=True) + + +class Goal(TimestampMixin, Base): + """An amount to reach, optionally by a date. Progress and the projected date are derived. + + `scope` is the same string the analytics API uses (`all`, `account:`, + `portfolio:`) so a goal can track one account or the whole net worth without a + second way of naming a subset. + """ + + __tablename__ = "goal" + + id: Mapped[int] = mapped_column(primary_key=True) + name: Mapped[str] = mapped_column(String(128), unique=True) + scope: Mapped[str] = mapped_column(String(32), default="all") + target_amount: Mapped[Decimal] + currency: Mapped[str] = mapped_column(String(3), default="RUB") + target_date: Mapped[date | None] + """NULL means «no deadline» — progress is still tracked, only the date is projected.""" + monthly_contribution: Mapped[Decimal | None] + """Planned top-up. Used for the projection when the goal has too little history for a + trailing XIRR to mean anything.""" + note: Mapped[str | None] = mapped_column(Text) + archived: Mapped[bool] = mapped_column(Boolean, default=False) diff --git a/backend/src/fintracker/models/pricing.py b/backend/src/fintracker/models/pricing.py index 07afbe3..8b2f18a 100644 --- a/backend/src/fintracker/models/pricing.py +++ b/backend/src/fintracker/models/pricing.py @@ -134,6 +134,29 @@ class CorporateAction(TimestampMixin, Base): source_id: Mapped[str | None] = mapped_column(String(128)) +class BondNominalSchedule(Base): + """What a bond's nominal becomes after each amortisation (plan §1.3). + + A bond quote is a percent of the *current* nominal, not the original one, so every + amortisation silently rewrites what a quote of «100» is worth. Without this schedule an + amortised bond keeps being valued at its issue par and the position drifts upward by + exactly the amount already repaid. The coupon forecast needs it for the same reason: a + coupon is a rate on the nominal that stands on the coupon date. + """ + + __tablename__ = "bond_nominal_schedule" + + instrument_id: Mapped[int] = mapped_column( + ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True + ) + effective_date: Mapped[date] = mapped_column(primary_key=True) + """First date this nominal applies to.""" + nominal: Mapped[Decimal] + currency: Mapped[str] = mapped_column(String(3)) + source: Mapped[str] = mapped_column(String(16)) + """moex | tinvest | ledger — `ledger` when derived from an amortisation payment.""" + + class PositionSnapshot(Base): """What the broker says a position is — kept for reconciliation, never for analytics.""" diff --git a/backend/src/fintracker/pricing/payouts.py b/backend/src/fintracker/pricing/payouts.py new file mode 100644 index 0000000..90caf90 --- /dev/null +++ b/backend/src/fintracker/pricing/payouts.py @@ -0,0 +1,441 @@ +"""Which of two payout feeds to believe, and where that choice is applied (plan §Фаза 4). + +Dividends, coupons and bond redemptions arrive from two independent feeds — T-Invest +(`sources/tinvest/sync_events.py`) and MOEX ISS (`sources/moex/payouts.py`) — and they +overlap. `corporate_action` is unique on `(instrument_id, kind, source, source_id)`, so both +feeds' rows coexist in the table by construction: a row cannot displace another source's row +even if we wanted it to. + +**Therefore priority is a READ rule, not a write rule.** Nothing is deleted or overwritten +when a feed syncs. The consumer — `analytics/income.py` — loads every `corporate_action` it +cares about and calls `resolve_payouts(actions)` exactly once, which collapses the duplicates +down to one winner per real-world payout. That placement is deliberate: + +* a write-time rule would have to delete the loser, and the next sync of the losing source + would insert it straight back — a table that never settles; +* the loser is the only thing that makes the winner checkable. Keeping both rows is what + lets `resolve_payouts` notice that the two feeds disagree on an amount and say so; +* the priority rule itself changes as we learn the feeds. A read rule changes behaviour on + the next refresh; a write rule would need a resync of years of history. + +The one thing that IS decided at write time is `bond_nominal_schedule`, whose primary key is +`(instrument_id, effective_date)` with `source` as a plain column — two feeds there really do +collide on one row, so `nominal_outranks()` below guards that upsert. + +## The priority itself + +Priority is keyed on the payout's `kind`, not on the instrument's asset class, because the +kind already says which world the payout lives in and — unlike the asset class — it is +carried on the row itself, which is what keeps this function free of the database. + +* **Coupons and bond redemptions: MOEX wins.** `/iss/securities/{secid}/bondization.json` is + the issuer's own schedule as registered with the exchange: every coupon to maturity, with + the date and the money per bond, plus the amortisation plan. T-Invest's `GetBondCoupons` + answers for a window and in practice carries the near coupons, which is enough for a + calendar and not enough for a forecast — and the value it reports for a floating-rate + coupon whose rate is not yet fixed is its own estimate, not a published figure. +* **Dividends: T-Invest wins.** `GetDividends` states the figure for the paper as the broker + will actually settle it, including the type of the payout (`dividend_type`) — for a + depositary receipt or a foreign issuer that is a different number from what the MOEX + register publishes. MOEX ISS's dividend endpoint is a register extract: right for the + ordinary share, but it knows nothing about which line of the paper is held. +* **A fact beats an announcement.** A row with `status = paid` (or `cancelled`) outranks any + `announced` or `forecast` row regardless of which feed it came from — that tier is checked + BEFORE the source tier. This is what keeps the `paid` rows that + `ledger/corporate_actions.py` derives from actual money movements from being shouted down + by a feed's announcement of the same payout. (Those rows are also safe at write time: the + feeds here never write `split`/`amortization`/`repayment` at all, because + `ledger.corporate_actions._prune` owns those kinds and deletes anything in them the ledger + does not imply. A feed's amortisation lands in `bond_nominal_schedule` instead.) +* **A disagreement is reported, never averaged.** When the loser states a different + `amount_per_unit` for the same `(instrument, kind, pay_date)`, the winner is still the + winner, and the gap goes to `FINDINGS` as `payout_amount_mismatch`. A systematic ~13 % gap + on dividends is the known one: T-Invest's `dividend_net` and the MOEX register are not + guaranteed to be on the same side of the withholding tax. + +`resolve_payouts` takes a plain sequence and touches no session, so `analytics/income.py` +can call it on whatever it has already loaded, and so it is testable without a database. +It reads only attributes (`instrument_id`, `kind`, `status`, `pay_date`, `ex_date`, +`record_date`, `amount_per_unit`, `currency`, `source`), so any row-like object will do. +""" + +from __future__ import annotations + +from collections.abc import Iterable, Sequence +from dataclasses import dataclass, field +from datetime import date +from decimal import Decimal +from typing import Any + +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import FINDINGS +from fintracker.models.pricing import ( + BondNominalSchedule, + CorporateAction, + CorporateActionKind, + CorporateActionStatus, +) + +TINVEST = "tinvest" +MOEX = "moex" + +#: Kinds that belong to a bond's own schedule — the ones MOEX `bondization` is authoritative on. +BOND_KINDS = frozenset( + { + CorporateActionKind.coupon, + CorporateActionKind.amortization, + CorporateActionKind.repayment, + CorporateActionKind.offer, + } +) + +#: Feed precedence per kind family, strongest first. Anything unlisted ranks below both. +BOND_SOURCE_ORDER = (MOEX, TINVEST) +EQUITY_SOURCE_ORDER = (TINVEST, MOEX) + +#: `bond_nominal_schedule` has one row per (instrument, date) whoever writes it, so this +#: precedence is applied at write time. `ledger` sits lowest only because it is derived from +#: a payment we saw rather than from the issuer's published plan — it is a fallback, not a lie. +NOMINAL_SOURCE_ORDER = (MOEX, TINVEST, "ledger") + +#: Below this the two feeds are rounding each other, not disagreeing. +ABS_TOLERANCE = Decimal("0.0001") +REL_TOLERANCE = Decimal("0.005") + +_STATUS_RANK = { + CorporateActionStatus.forecast: 0, + CorporateActionStatus.announced: 1, + # both are settled facts: the money moved, or the issuer called the payout off + CorporateActionStatus.cancelled: 2, + CorporateActionStatus.paid: 2, +} + + +#: The attributes a row has to carry to be resolvable. Kept as a comment rather than a +#: `Protocol` bound on purpose: pyright matches a protocol on an attribute's DECLARED type, +#: and every column of `CorporateAction` is declared `Mapped[...]`, so the model — the one +#: type this function actually exists for — would fail the bound it is meant to describe. +PAYOUT_ATTRIBUTES = ( + "instrument_id", + "kind", + "status", + "record_date", + "ex_date", + "pay_date", + "amount_per_unit", + "currency", + "source", +) + + +@dataclass(frozen=True) +class PayoutRow: + """One payout as a feed produced it, before it becomes a `corporate_action` row. + + Shared by both sources so that the two syncs cannot drift apart on what a payout is, + and so the mapping from a feed record to this shape stays a pure function. + """ + + instrument_id: int + kind: CorporateActionKind + status: CorporateActionStatus + source: str + source_id: str + record_date: date | None = None + ex_date: date | None = None + pay_date: date | None = None + amount_per_unit: Decimal | None = None + currency: str | None = None + + def as_values(self) -> dict[str, Any]: + """The `corporate_action` column map for an insert.""" + return { + "instrument_id": self.instrument_id, + "kind": self.kind, + "status": self.status, + "record_date": self.record_date, + "ex_date": self.ex_date, + "pay_date": self.pay_date, + "amount_per_unit": self.amount_per_unit, + "currency": self.currency, + "ratio": None, + "source": self.source, + "source_id": self.source_id, + } + + +@dataclass(frozen=True) +class NominalPoint: + """What a bond's nominal becomes on `effective_date`, per `bond_nominal_schedule`.""" + + instrument_id: int + effective_date: date + nominal: Decimal + currency: str + source: str + + def as_values(self) -> dict[str, Any]: + return { + "instrument_id": self.instrument_id, + "effective_date": self.effective_date, + "nominal": self.nominal, + "currency": self.currency, + "source": self.source, + } + + +@dataclass(frozen=True) +class PayoutConflict: + """Two feeds naming different money for one payout.""" + + instrument_id: int + kind: CorporateActionKind + day: date | None + winner: Any + loser: Any + + @property + def detail(self) -> str: + return ( + f"инструмент {self.instrument_id}, {self.kind}, {self.day}: " + f"{self.winner.source} даёт {self.winner.amount_per_unit}, " + f"{self.loser.source} — {self.loser.amount_per_unit}" + ) + + +@dataclass +class Resolution: + payouts: list[Any] = field(default_factory=list) + conflicts: list[PayoutConflict] = field(default_factory=list) + + +def source_order(kind: Any) -> tuple[str, ...]: + """Feed precedence for one kind of payout, strongest first.""" + return BOND_SOURCE_ORDER if kind in BOND_KINDS else EQUITY_SOURCE_ORDER + + +def source_rank(kind: Any, source: str) -> int: + """Higher is stronger. An unknown feed ranks below every known one, never above.""" + order = source_order(kind) + return len(order) - order.index(source) if source in order else 0 + + +def status_rank(status: Any) -> int: + return _STATUS_RANK.get(status, 0) + + +def nominal_outranks(incoming: str, existing: str) -> bool: + """May a `bond_nominal_schedule` row from `incoming` replace one from `existing`? + + Equal sources count as outranking themselves: a feed must be able to correct its own + earlier answer, or a revised amortisation plan would never land. + """ + return _nominal_rank(incoming) >= _nominal_rank(existing) + + +def weaker_or_equal_nominal_sources(incoming: str) -> list[str]: + """Sources an `incoming` row may overwrite — the guard list for the upsert's WHERE.""" + rank = _nominal_rank(incoming) + return [s for s in NOMINAL_SOURCE_ORDER if _nominal_rank(s) <= rank] + + +def _nominal_rank(source: str) -> int: + order = NOMINAL_SOURCE_ORDER + return len(order) - order.index(source) if source in order else 0 + + +def resolve[P: Any](actions: Iterable[P]) -> Resolution: + """Collapse overlapping feed rows to one payout each, pure and database-free. + + Rows are grouped per `(instrument_id, kind)` and then per payout by `_same_payout`, + which pairs rows that share any stated date. Within a group the winner is decided by + the fact tier first and the feed precedence second; the losers are kept only long + enough to check them against the winner's amount. + """ + result = Resolution() + by_paper: dict[tuple[int, Any], list[P]] = {} + for action in actions: + by_paper.setdefault((action.instrument_id, action.kind), []).append(action) + + for (instrument_id, kind), members in by_paper.items(): + for group in _same_payout(members): + ranked = sorted(group, key=_rank_key, reverse=True) + winner = ranked[0] + result.payouts.append(winner) + day = payout_day(winner) + result.conflicts += [ + PayoutConflict( + instrument_id=instrument_id, kind=kind, day=day, winner=winner, loser=loser + ) + for loser in ranked[1:] + if _disagrees(winner, loser) + ] + result.payouts.sort(key=lambda a: (payout_day(a) or date.min, a.instrument_id, str(a.kind))) + return result + + +def _same_payout[P: Any](members: Sequence[P]) -> list[list[P]]: + """Split one paper's rows of one kind into groups that describe the same payout. + + Two rows are the same payout when any of their stated dates coincide — pay, ex or + record. Matching on a single chosen date would not work across these two feeds: MOEX's + dividend extract states only the register-closing date while T-Invest states the record + AND the payment date, so keying on the payment date leaves them in separate buckets and + the income total counts the dividend twice. Two genuinely different payouts of one paper + never share a date, which is what makes the overlap safe. + + A row with no date at all joins nothing: without a date there is no evidence it is the + same payout as anything else, and merging on the strength of the instrument alone would + silently drop money. + """ + groups: list[tuple[set[date], list[P]]] = [] + for action in members: + days = {d for d in (action.pay_date, action.ex_date, action.record_date) if d} + if not days: + groups.append((set(), [action])) + continue + hits = [g for g in groups if g[0] & days] + merged_days = set(days) + merged_rows = [action] + for group in hits: + merged_days |= group[0] + merged_rows += group[1] + groups.remove(group) + groups.append((merged_days, merged_rows)) + return [rows for _, rows in groups] + + +def resolve_payouts[P: Any](actions: Sequence[P], *, report: bool = True) -> list[P]: + """The contract `analytics/income.py` calls: one row per real payout, duplicates dropped. + + `report=False` turns off the `FINDINGS` side effect, for callers that only want the + resolution (a preview endpoint, a test). The resolution itself is always pure. + """ + result = resolve(actions) + if report and result.conflicts: + _report(result.conflicts) + return result.payouts + + +def payout_day(action: Any) -> date | None: + """The date two feeds can be compared on: pay date, else ex-date, else record date.""" + return action.pay_date or action.ex_date or action.record_date + + +def _rank_key(action: Any) -> tuple[int, int, int, str]: + """Fact tier first, then feed precedence — a stated amount breaks a remaining tie.""" + return ( + status_rank(action.status), + source_rank(action.kind, action.source), + 0 if action.amount_per_unit is None else 1, + action.source or "", + ) + + +def _disagrees(winner: Any, loser: Any) -> bool: + """True when both feeds named an amount and the gap is more than rounding. + + A missing amount on either side is a gap, not a disagreement: one feed simply has not + published the figure yet (a floating coupon whose rate is unfixed), and calling that a + conflict would fill the quality report with noise on every refresh. + """ + a, b = winner.amount_per_unit, loser.amount_per_unit + if a is None or b is None: + return False + if (winner.currency or "") != (loser.currency or ""): + return True + return abs(a - b) > max(ABS_TOLERANCE, abs(a) * REL_TOLERANCE) + + +def _report(conflicts: Sequence[PayoutConflict]) -> None: + FINDINGS.add( + "payout_amount_mismatch", + "warn", + f"Источники расходятся в сумме выплаты по {len(conflicts)} записям " + f"(взята запись по приоритету): {'; '.join(c.detail for c in conflicts[:5])}", + count=len(conflicts), + ref={"instruments": sorted({c.instrument_id for c in conflicts})}, + ) + + +async def upsert_payouts(session: AsyncSession, rows: Sequence[PayoutRow]) -> int: + """Write feed payouts idempotently on `(instrument_id, kind, source, source_id)`. + + Every feed goes through here, so nothing else has to remember that a re-sync of the same + window must be a no-op, or that a row belonging to another `source` is never touched. + """ + if not rows: + return 0 + # one statement cannot address the same key twice; the last mapping of a key wins + unique = {(r.instrument_id, r.kind, r.source, r.source_id): r for r in rows} + stmt = pg_insert(CorporateAction).values([r.as_values() for r in unique.values()]) + await session.execute( + stmt.on_conflict_do_update( + index_elements=["instrument_id", "kind", "source", "source_id"], + set_={ + "status": stmt.excluded.status, + "record_date": stmt.excluded.record_date, + "ex_date": stmt.excluded.ex_date, + "pay_date": stmt.excluded.pay_date, + "amount_per_unit": stmt.excluded.amount_per_unit, + "currency": stmt.excluded.currency, + }, + ) + ) + return len(unique) + + +async def upsert_nominals(session: AsyncSession, points: Sequence[NominalPoint]) -> int: + """Write a bond's nominal schedule, letting only an equal-or-stronger source overwrite. + + This is the one place a payout feed decides a precedence at write time, and it has to: + `bond_nominal_schedule` is keyed on `(instrument_id, effective_date)` with `source` as a + plain column, so MOEX's published amortisation plan and T-Invest's redemption events land + on the very same row. See `nominal_outranks`. + """ + if not points: + return 0 + unique = {(p.instrument_id, p.effective_date): p for p in points} + written = 0 + by_source: dict[str, list[NominalPoint]] = {} + for point in unique.values(): + by_source.setdefault(point.source, []).append(point) + for source, batch in by_source.items(): + stmt = pg_insert(BondNominalSchedule).values([p.as_values() for p in batch]) + await session.execute( + stmt.on_conflict_do_update( + index_elements=["instrument_id", "effective_date"], + set_={ + "nominal": stmt.excluded.nominal, + "currency": stmt.excluded.currency, + "source": stmt.excluded.source, + }, + where=BondNominalSchedule.source.in_(weaker_or_equal_nominal_sources(source)), + ) + ) + written += len(batch) + return written + + +__all__ = [ + "BOND_KINDS", + "MOEX", + "NOMINAL_SOURCE_ORDER", + "TINVEST", + "NominalPoint", + "PayoutConflict", + "PayoutRow", + "Resolution", + "nominal_outranks", + "payout_day", + "resolve", + "resolve_payouts", + "source_order", + "source_rank", + "status_rank", + "upsert_nominals", + "upsert_payouts", + "weaker_or_equal_nominal_sources", +] diff --git a/backend/src/fintracker/sources/moex/__init__.py b/backend/src/fintracker/sources/moex/__init__.py index fde62df..d54372b 100644 --- a/backend/src/fintracker/sources/moex/__init__.py +++ b/backend/src/fintracker/sources/moex/__init__.py @@ -1,8 +1,10 @@ """The `moex` source: prices and bond schedules from the MOEX ISS.""" +from fintracker.sources.moex.payouts import MoexPayoutsSource from fintracker.sources.moex.sync import MoexSource from fintracker.sources.registry import register register(MoexSource()) +register(MoexPayoutsSource()) -__all__ = ["MoexSource"] +__all__ = ["MoexPayoutsSource", "MoexSource"] diff --git a/backend/src/fintracker/sources/moex/payouts.py b/backend/src/fintracker/sources/moex/payouts.py new file mode 100644 index 0000000..aaef7d2 --- /dev/null +++ b/backend/src/fintracker/sources/moex/payouts.py @@ -0,0 +1,354 @@ +"""The `moex_payouts` source: the second payout feed, from MOEX ISS (plan §Фаза 4). + +Two endpoints, both free and keyless: + +* `/iss/securities/{secid}/bondization.json` — the bond's registered schedule: every coupon + to maturity and the amortisation plan. `MoexClient.bondization` already reads it. +* `/iss/securities/{secid}/dividends.json` — the dividend register extract for a share. + Only this module uses it, so the request lives here rather than on the client. + +What it writes, and what it deliberately does not: + +* coupons -> `corporate_action(kind=coupon, source='moex')`, alongside the T-Invest rows + for the same coupons rather than instead of them. Which one an analytic reads is decided + by `pricing/payouts.resolve_payouts`, at read time — see that module for why. +* dividends -> `corporate_action(kind=dividend, source='moex')`, same arrangement. +* amortisations -> `bond_nominal_schedule(source='moex')`, and NOT + `corporate_action(kind=amortization)`: that kind belongs to `ledger/corporate_actions.py`, + whose prune deletes every row in it the ledger does not imply. + +**The amortisation plan is read as a run-out, not as a column.** ISS states `value` (repaid +per bond) and `facevalue` per row, but which side of the payment `facevalue` stands on is not +documented and differs between papers. Summing what is still to be repaid is unambiguous: +everything the issuer will ever repay per bond is the nominal, so the nominal standing after +a given amortisation is the sum of those after it. `facevalue` is used only as a cross-check, +and a mismatch is a warning rather than a different answer. + +**There is no raw tier here.** `raw_moex_doc` (plan §1.1) does not exist yet, and the MOEX +source has never had one: ISS answers are cheap to re-fetch, unlike the rate-limited +T-Invest feeds the raw tables exist for. +""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from dataclasses import dataclass +from datetime import date +from decimal import Decimal + +import httpx +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import today_local +from fintracker.models import AssetClass, Event, EventStatus, Instrument +from fintracker.models.pricing import CorporateActionKind, CorporateActionStatus +from fintracker.pricing.payouts import ( + MOEX, + NominalPoint, + PayoutRow, + upsert_nominals, + upsert_payouts, +) +from fintracker.sources.base import SyncContext, SyncResult +from fintracker.sources.moex.client import ( + BASE, + AmortisationRow, + CouponRow, + MoexClient, + MoexError, + _currency, + _date, + _decimal, + _rows, + new_http_client, +) +from fintracker.sources.moex.sync import _secid_candidates + +log = logging.getLogger(__name__) + +SOURCE = MOEX +NAME = "moex_payouts" + +BOND_CLASSES = frozenset({AssetClass.bond}) +DIVIDEND_CLASSES = frozenset({AssetClass.share, AssetClass.etf, AssetClass.fund}) + +ZERO = Decimal(0) +NOMINAL_TOLERANCE = Decimal("0.01") +"""Below this the run-out and the stated face value are rounding each other.""" + + +@dataclass(frozen=True) +class MoexDividendRow: + """One row of `/iss/securities/{secid}/dividends.json`. + + The extract is a register snapshot: it states when the register closed and how much per + share was declared, and nothing about when the money is actually paid — which is the + gap T-Invest fills, and one of the reasons dividends are read from T-Invest first. + """ + + secid: str + registry_close_date: date | None + value: Decimal | None + currency: str | None + + +@dataclass(frozen=True) +class Target: + instrument_id: int + secid: str + asset_class: AssetClass + currency: str + + +async def fetch_dividends(client: httpx.AsyncClient, secid: str) -> list[MoexDividendRow]: + """The dividend register extract for one paper; an unlisted paper answers with nothing.""" + response = await client.get( + f"{BASE}/securities/{secid}/dividends.json", params={"iss.meta": "off"} + ) + if response.status_code == httpx.codes.NOT_FOUND: + raise MoexError(f"dividends for {secid} not found") + response.raise_for_status() + return [ + MoexDividendRow( + secid=str(row.get("secid") or secid), + registry_close_date=_date(row.get("registryclosedate")), + value=_decimal(row.get("value")), + currency=_currency(row.get("currencyid")), + ) + for row in _rows(response.json(), "dividends") + ] + + +def coupon_payout(row: CouponRow, *, instrument_id: int, today: date) -> PayoutRow | None: + """One bondization coupon as a `corporate_action` row. + + A coupon whose rate is not fixed yet arrives dated but with no `value`; it is kept, with + a NULL amount, because the date is real and the calendar needs it. `resolve_payouts` + treats a missing amount as a gap rather than a disagreement for exactly this case. + """ + if row.coupon_date is None: + return None + return PayoutRow( + instrument_id=instrument_id, + kind=CorporateActionKind.coupon, + status=_status(row.coupon_date, today), + source=SOURCE, + source_id=f"cpn:{row.coupon_date.isoformat()}", + pay_date=row.coupon_date, + amount_per_unit=row.value, + currency=(row.currency or "").upper() or None, + ) + + +def dividend_payout(row: MoexDividendRow, *, instrument_id: int, today: date) -> PayoutRow | None: + """One register row as a `corporate_action` row. + + The register-closing date is the record date, and it is the only date stated — so it is + also what a date-based merge with the T-Invest row keys on. + """ + if row.registry_close_date is None: + return None + return PayoutRow( + instrument_id=instrument_id, + kind=CorporateActionKind.dividend, + status=_status(row.registry_close_date, today), + source=SOURCE, + source_id=f"div:{row.registry_close_date.isoformat()}", + record_date=row.registry_close_date, + amount_per_unit=row.value, + currency=(row.currency or "").upper() or None, + ) + + +def nominal_schedule( + amortisations: Sequence[AmortisationRow], + *, + instrument_id: int, + currency: str, + warnings: list[str], + secid: str = "", +) -> list[NominalPoint]: + """The nominal standing after each amortisation, run out from the total still to repay.""" + usable = sorted( + (a for a in amortisations if a.amort_date is not None and a.value is not None), + key=lambda a: a.amort_date or date.min, + ) + if not usable: + return [] + + total = sum((a.value or ZERO for a in usable), start=ZERO) + stated = max((a.face_value for a in usable if a.face_value is not None), default=None) + if stated is not None and abs(stated - total) > NOMINAL_TOLERANCE: + warnings.append( + f"{secid}: сумма амортизаций {total} расходится с номиналом {stated} " + "— график построен по сумме выплат" + ) + + remaining = total + points: list[NominalPoint] = [] + for row in usable: + remaining -= row.value or ZERO + points.append( + NominalPoint( + instrument_id=instrument_id, + effective_date=row.amort_date or date.min, + nominal=remaining, + currency=(row.currency or currency).upper(), + source=SOURCE, + ) + ) + return points + + +def _status(day: date, today: date) -> CorporateActionStatus: + """Past dates are facts the issuer has settled; future ones are announcements.""" + return CorporateActionStatus.paid if day < today else CorporateActionStatus.announced + + +class MoexPayoutsSource: + """`Source` protocol, same shape as `MoexSource`. Registration is done elsewhere.""" + + name = NAME + + async def sync(self, ctx: SyncContext) -> SyncResult: + session = ctx.session + today = today_local() + targets = await load_targets(session) + if not targets: + log.info("moex_payouts: no priceable instruments in the ledger yet") + return SyncResult(cursor_after=today.isoformat(), counts={"payouts": 0}, changed=False) + + counts = {"instruments": 0, "coupons": 0, "dividends": 0, "payouts": 0, "nominals": 0} + warnings: list[str] = [] + payouts: list[PayoutRow] = [] + nominals: list[NominalPoint] = [] + + async with new_http_client() as http, MoexClient(http) as moex: + for target in targets: + counts["instruments"] += 1 + if target.asset_class in BOND_CLASSES: + rows = await self._bond(moex, target, today, payouts, nominals, warnings) + counts["coupons"] += rows + else: + counts["dividends"] += await self._dividends( + http, target, today, payouts, warnings + ) + + counts["payouts"] = await upsert_payouts(session, payouts) + counts["nominals"] = await upsert_nominals(session, nominals) + await session.commit() + + log.info( + "moex_payouts: %s instruments, %s payouts, %s nominal points", + counts["instruments"], + counts["payouts"], + counts["nominals"], + ) + return SyncResult( + cursor_after=today.isoformat(), + counts=counts, + warnings=warnings, + changed=bool(counts["payouts"] or counts["nominals"]), + ) + + async def _bond( + self, + moex: MoexClient, + target: Target, + today: date, + payouts: list[PayoutRow], + nominals: list[NominalPoint], + warnings: list[str], + ) -> int: + coupons: list[CouponRow] = [] + amortisations: list[AmortisationRow] = [] + for secid in _secid_candidates(target.secid): + try: + coupons, amortisations = await moex.bondization(secid) + except (MoexError, httpx.HTTPError) as err: + warnings.append(f"{secid}: {err}") + continue + if coupons or amortisations: + break + payouts += [ + payout + for coupon in coupons + if (payout := coupon_payout(coupon, instrument_id=target.instrument_id, today=today)) + ] + nominals += nominal_schedule( + amortisations, + instrument_id=target.instrument_id, + currency=target.currency, + warnings=warnings, + secid=target.secid, + ) + return len(coupons) + + async def _dividends( + self, + http: httpx.AsyncClient, + target: Target, + today: date, + payouts: list[PayoutRow], + warnings: list[str], + ) -> int: + rows: list[MoexDividendRow] = [] + for secid in _secid_candidates(target.secid): + try: + rows = await fetch_dividends(http, secid) + except (MoexError, httpx.HTTPError) as err: + warnings.append(f"{secid}: {err}") + continue + if rows: + break + payouts += [ + payout + for row in rows + if (payout := dividend_payout(row, instrument_id=target.instrument_id, today=today)) + ] + return len(rows) + + +async def load_targets(session: AsyncSession) -> list[Target]: + """Instruments the confirmed ledger touches that MOEX can answer about at all.""" + rows = ( + await session.execute( + select( + Instrument.id, + Instrument.ticker, + Instrument.asset_class, + Instrument.currency, + ) + .join(Event, Event.instrument_id == Instrument.id) + .where( + Event.status == EventStatus.confirmed, + Instrument.ticker.is_not(None), + Instrument.asset_class.in_(BOND_CLASSES | DIVIDEND_CLASSES), + ) + .group_by(Instrument.id, Instrument.ticker, Instrument.asset_class, Instrument.currency) + ) + ).all() + return [ + Target( + instrument_id=r.id, + secid=r.ticker, + asset_class=r.asset_class, + currency=r.currency, + ) + for r in rows + ] + + +__all__ = [ + "NAME", + "MoexDividendRow", + "MoexPayoutsSource", + "Target", + "coupon_payout", + "dividend_payout", + "fetch_dividends", + "load_targets", + "nominal_schedule", +] diff --git a/backend/src/fintracker/sources/tinvest/__init__.py b/backend/src/fintracker/sources/tinvest/__init__.py index 381b8d3..c564e5e 100644 --- a/backend/src/fintracker/sources/tinvest/__init__.py +++ b/backend/src/fintracker/sources/tinvest/__init__.py @@ -2,7 +2,9 @@ from fintracker.sources.registry import register from fintracker.sources.tinvest.sync import TinvestSource +from fintracker.sources.tinvest.sync_events import TinvestEventsSource register(TinvestSource()) +register(TinvestEventsSource()) -__all__ = ["TinvestSource"] +__all__ = ["TinvestEventsSource", "TinvestSource"] diff --git a/backend/src/fintracker/sources/tinvest/client.py b/backend/src/fintracker/sources/tinvest/client.py index f02b2d0..bd502f3 100644 --- a/backend/src/fintracker/sources/tinvest/client.py +++ b/backend/src/fintracker/sources/tinvest/client.py @@ -155,6 +155,61 @@ class InstrumentInfo: payload: dict[str, Any] +@dataclass(frozen=True) +class DividendRow: + """One `Dividend` from GetDividends, flattened. + + `instrument_uid` is the uid we ASKED for: the record itself carries no instrument id + at all, so the only link back to the paper is the request. + """ + + instrument_uid: str + amount: Decimal | None + """`dividend_net` — the per-share figure T-Invest publishes.""" + currency: str | None + payment_date: datetime | None + declared_date: datetime | None + record_date: datetime | None + last_buy_date: datetime | None + """Last day a purchase still earns the dividend; the ex-date is the next trading day.""" + dividend_type: str | None + regularity: str | None + payload: dict[str, Any] + + +@dataclass(frozen=True) +class BondCouponRow: + """One `Coupon` from GetBondCoupons.""" + + instrument_uid: str + coupon_number: int | None + coupon_date: datetime | None + fix_date: datetime | None + pay_one_bond: Decimal | None + currency: str | None + coupon_type: str + """The enum's NAME, e.g. COUPON_TYPE_CONSTANT — an unknown one is the caller's call.""" + coupon_period: int | None + payload: dict[str, Any] + + +@dataclass(frozen=True) +class BondEventRow: + """One `BondEvent` from GetBondEvents (coupons, calls, redemptions).""" + + instrument_uid: str + event_type: str + """The enum's NAME: EVENT_TYPE_CPN | EVENT_TYPE_CALL | EVENT_TYPE_MTY | EVENT_TYPE_CONV.""" + event_number: int | None + event_date: datetime | None + fix_date: datetime | None + pay_date: datetime | None + pay_one_bond: Decimal | None + """Money paid per bond — for a redemption event this is the principal repaid.""" + currency: str | None + payload: dict[str, Any] + + def _as_dict(message: Any) -> dict[str, Any]: """A JSON-able dict for the `raw_*` tables. @@ -436,6 +491,125 @@ class TinvestClient: country=info.country or match.country, ) + async def dividends( + self, instrument_uid: str, *, since: datetime, until: datetime + ) -> list[DividendRow]: + """GetDividends over [since, until] for one paper. + + Only shares and depositary receipts have a dividend history here; asking about a + bond, an ETF or a currency answers NOT_FOUND or an empty list, which is a fact + about the paper rather than an error — hence the empty list instead of a raise. + """ + resp = await self._payout_call( + "GetDividends", + instrument_uid, + lambda: self._client.instruments.get_dividends( + instrument_id=instrument_uid, from_=since, to=until + ), + ) + if resp is None: + return [] + return [ + DividendRow( + instrument_uid=instrument_uid, + amount=_money(d.dividend_net), + currency=_currency(d.dividend_net), + payment_date=getattr(d, "payment_date", None), + declared_date=getattr(d, "declared_date", None), + record_date=getattr(d, "record_date", None), + last_buy_date=getattr(d, "last_buy_date", None), + dividend_type=getattr(d, "dividend_type", None) or None, + regularity=getattr(d, "regularity", None) or None, + payload=_as_dict(d), + ) + for d in resp.dividends + ] + + async def bond_coupons( + self, instrument_uid: str, *, since: datetime, until: datetime + ) -> list[BondCouponRow]: + """GetBondCoupons: the coupon schedule T-Invest holds for a bond.""" + resp = await self._payout_call( + "GetBondCoupons", + instrument_uid, + lambda: self._client.instruments.get_bond_coupons( + instrument_id=instrument_uid, from_=since, to=until + ), + ) + if resp is None: + return [] + return [ + BondCouponRow( + instrument_uid=instrument_uid, + coupon_number=int(getattr(c, "coupon_number", 0) or 0) or None, + coupon_date=getattr(c, "coupon_date", None), + fix_date=getattr(c, "fix_date", None), + pay_one_bond=_money(c.pay_one_bond), + currency=_currency(c.pay_one_bond), + coupon_type=getattr(getattr(c, "coupon_type", None), "name", "") or "", + coupon_period=int(getattr(c, "coupon_period", 0) or 0) or None, + payload=_as_dict(c), + ) + for c in resp.events + ] + + async def bond_events( + self, instrument_uid: str, *, since: datetime, until: datetime, event_type: str + ) -> list[BondEventRow]: + """GetBondEvents of one `EventType` name (EVENT_TYPE_CPN, EVENT_TYPE_MTY, …). + + The request takes exactly one type, so a caller after both coupons and redemptions + pays two RPCs. Amortisation has no type of its own: a partially amortised bond + reports several `EVENT_TYPE_MTY` events, each repaying a slice of the principal. + """ + # `t_tech.invest` re-exports neither of these — they live only in `.schemas` + from t_tech.invest.schemas import EventType, GetBondEventsRequest + + try: + kind = EventType[event_type] + except KeyError: + raise ValueError(f"unknown bond EventType: {event_type}") from None + resp = await self._payout_call( + "GetBondEvents", + instrument_uid, + lambda: self._client.instruments.get_bond_events( + GetBondEventsRequest(instrument_id=instrument_uid, from_=since, to=until, type=kind) + ), + ) + if resp is None: + return [] + return [ + BondEventRow( + instrument_uid=instrument_uid, + event_type=getattr(getattr(e, "event_type", None), "name", "") or "", + event_number=int(getattr(e, "event_number", 0) or 0) or None, + event_date=getattr(e, "event_date", None), + fix_date=getattr(e, "fix_date", None), + pay_date=getattr(e, "pay_date", None) or getattr(e, "real_pay_date", None), + pay_one_bond=_money(e.pay_one_bond), + currency=_currency(e.pay_one_bond), + payload=_as_dict(e), + ) + for e in resp.events + ] + + async def _payout_call( + self, rpc: str, instrument_uid: str, fn: Callable[[], Coroutine[Any, Any, T]] + ) -> T | None: + """`_call`, but a paper the RPC does not serve yields None instead of failing the run. + + The payout RPCs are typed to an asset class: a share has no coupon schedule and an + ETF has no dividend history in this feed, and both answer NOT_FOUND or + INVALID_ARGUMENT. One such paper must not abort a sync over the whole portfolio. + """ + try: + return await self._call(fn) + except AioRequestError as err: + if err.code in (StatusCode.NOT_FOUND, StatusCode.INVALID_ARGUMENT): + log.info("tinvest: %s has no %s data", instrument_uid, rpc) + return None + raise + def _silence_sdk_telemetry() -> None: """Stop the SDK from reporting our errors to T-Bank's Sentry. diff --git a/backend/src/fintracker/sources/tinvest/sync_events.py b/backend/src/fintracker/sources/tinvest/sync_events.py new file mode 100644 index 0000000..1064c09 --- /dev/null +++ b/backend/src/fintracker/sources/tinvest/sync_events.py @@ -0,0 +1,463 @@ +"""The `tinvest_events` source: the payout calendar as T-Invest publishes it (plan §Фаза 4). + +One run, per instrument the ledger has ever touched: + +1. shares, ETFs -> `GetDividends` -> `raw_tinvest_event('dividend')` -> `corporate_action` +2. bonds -> `GetBondCoupons` -> `raw_tinvest_event('coupon')` -> `corporate_action` +3. bonds -> `GetBondEvents` -> `raw_tinvest_event('bond_event')` -> `bond_nominal_schedule` + +**Scope comes from the ledger, not from the current portfolio.** A paper that was sold last +year still paid dividends while it was held, and the income history has to keep them. + +**The window is the paper's whole life, not an increment.** These feeds are schedules, not +streams: a coupon plan is revised in place (a floating rate gets fixed, a date moves), and an +incremental read would keep an obsolete row forever. The cost is bounded — one RPC per share, +two per bond, on a portfolio of under a hundred papers — and `client._call` waits out the +Instruments limit of 200/min on its own, so a full sweep is slow rather than fragile. + +**Amortisation does not go into `corporate_action`.** `ledger/corporate_actions.py` owns the +kinds `split`, `amortization` and `repayment` end to end: its `_prune` deletes every row in +those kinds the ledger does not imply, so anything this module wrote there would survive +until the next refresh and no longer. What a feed knows that the ledger cannot is the nominal +the bond carries *before* the money arrives, and that is exactly what +`bond_nominal_schedule` is for — the amortisation cash is the difference between two +consecutive nominals. Dividends and coupons are untouched by that prune and are written +normally, as `announced` ahead of the pay date and `paid` once it has passed. + +**Redemption events carry no nominal, only money.** `GetBondEvents(EVENT_TYPE_MTY)` returns +one event per partial redemption with `pay_one_bond` — the slice of principal repaid. The +schedule is reconstructed by running that backwards from the total: everything the issuer +will ever repay per bond IS the original nominal, so the nominal standing after each +redemption is the sum of the redemptions still to come. Reading `instrument.nominal` instead +would not work: T-Invest reports the *current* nominal, which is already amortised down. +""" + +from __future__ import annotations + +import logging +from collections.abc import Sequence +from datetime import UTC, date, datetime +from decimal import Decimal +from typing import Any +from zoneinfo import ZoneInfo + +from sqlalchemy import select +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from fintracker.analytics import today_local +from fintracker.models import AssetClass, Event, EventStatus, Instrument, RawTinvestEvent +from fintracker.models.pricing import CorporateActionKind, CorporateActionStatus +from fintracker.pricing.payouts import ( + TINVEST, + NominalPoint, + PayoutRow, + upsert_nominals, + upsert_payouts, +) +from fintracker.sources.base import SyncContext, SyncResult +from fintracker.sources.tinvest.client import ( + BondCouponRow, + BondEventRow, + DividendRow, + TinvestClient, +) +from fintracker.sources.tinvest.sync import TinvestAuthError + +log = logging.getLogger(__name__) + +SOURCE = TINVEST +NAME = "tinvest_events" +MSK = ZoneInfo("Europe/Moscow") + +HISTORY_START = datetime(2015, 1, 1, tzinfo=UTC) +"""Far enough back for any paper the portfolio has held; the API clamps to the issue date.""" +FORWARD_YEARS = 10 +"""How far ahead to ask. A coupon plan runs to maturity, and a long OFZ is a decade out.""" + +#: Which asset classes have a dividend history in this feed at all. +DIVIDEND_CLASSES = frozenset({AssetClass.share, AssetClass.etf, AssetClass.fund}) +BOND_CLASSES = frozenset({AssetClass.bond}) + +#: Coupon types the API states. `UNSPECIFIED` is not one of them — it is the API declining +#: to say, and a payout whose nature is unknown is warned about, never filed as "other". +KNOWN_COUPON_TYPES = frozenset( + { + "COUPON_TYPE_CONSTANT", + "COUPON_TYPE_FLOATING", + "COUPON_TYPE_DISCOUNT", + "COUPON_TYPE_MORTGAGE", + "COUPON_TYPE_FIX", + "COUPON_TYPE_VARIABLE", + "COUPON_TYPE_OTHER", + } +) + +REDEMPTION = "EVENT_TYPE_MTY" +"""The only bond event type this sync asks for: partial and final redemptions.""" + +ZERO = Decimal(0) + + +class Target: + """One instrument to ask about, with the identity and shape the feeds need.""" + + __slots__ = ("asset_class", "currency", "instrument_id", "ticker", "uid") + + def __init__( + self, + instrument_id: int, + uid: str, + asset_class: AssetClass, + currency: str, + ticker: str | None, + ) -> None: + self.instrument_id = instrument_id + self.uid = uid + self.asset_class = asset_class + self.currency = currency + self.ticker = ticker + + +def msk_date(value: datetime | None) -> date | None: + """A feed timestamp as the trading day it belongs to (conventions: trade dates in MSK). + + The API stamps these at midnight UTC, which is the previous evening in Moscow — reading + the date off the UTC value moves every payout one day earlier. + """ + if value is None: + return None + if value.tzinfo is None: + value = value.replace(tzinfo=UTC) + return value.astimezone(MSK).date() + + +def status_for(day: date | None, today: date) -> CorporateActionStatus: + """A payout whose date has passed is `paid`; one still ahead is `announced`. + + The feed states the issuer's schedule, not our bank statement, so `paid` here means "the + issuer paid it", which is what the calendar and the income forecast need. Whether the + money reached a specific account is the ledger's answer, and the ledger's `paid` rows + outrank these — see `pricing/payouts.py`. + """ + if day is not None and day < today: + return CorporateActionStatus.paid + return CorporateActionStatus.announced + + +def dividend_payout(row: DividendRow, *, instrument_id: int, today: date) -> PayoutRow | None: + """One `GetDividends` record as a `corporate_action` row, or None when it says nothing. + + `last_buy_date` is the last day a purchase still earns the dividend, which is the day + BEFORE the ex-date — but it is the only ex-side date the feed states, and storing it as + `ex_date` keeps the calendar honest to within one trading day. The record date is the + one that decides entitlement, and it is stated exactly. + """ + record = msk_date(row.record_date) + pay = msk_date(row.payment_date) + declared = msk_date(row.declared_date) + key = record or pay or declared + if key is None: + return None + return PayoutRow( + instrument_id=instrument_id, + kind=CorporateActionKind.dividend, + status=status_for(pay or record, today), + source=SOURCE, + source_id=f"div:{key.isoformat()}", + record_date=record, + ex_date=msk_date(row.last_buy_date), + pay_date=pay, + amount_per_unit=row.amount, + currency=row.currency, + ) + + +def coupon_payout( + row: BondCouponRow, *, instrument_id: int, today: date, warnings: list[str] +) -> PayoutRow | None: + """One `GetBondCoupons` record as a `corporate_action` row. + + A zero payment is dropped without a warning: that is a discount bond's nominal coupon, + not a payout. An unrecognised `coupon_type` IS warned about and dropped — filing it as + a plain coupon would put money of an unknown nature into the income forecast. + """ + if row.coupon_type not in KNOWN_COUPON_TYPES: + warnings.append( + f"{row.instrument_uid}: незнакомый тип купона {row.coupon_type or '<пусто>'} — пропущен" + ) + return None + day = msk_date(row.coupon_date) + if day is None: + return None + if row.pay_one_bond is not None and row.pay_one_bond == ZERO: + return None + key = row.coupon_number if row.coupon_number else day.isoformat() + return PayoutRow( + instrument_id=instrument_id, + kind=CorporateActionKind.coupon, + status=status_for(day, today), + source=SOURCE, + source_id=f"cpn:{key}", + record_date=msk_date(row.fix_date), + ex_date=None, + pay_date=day, + amount_per_unit=row.pay_one_bond, + currency=row.currency, + ) + + +def nominal_schedule( + events: Sequence[BondEventRow], + *, + instrument_id: int, + currency: str, + warnings: list[str], +) -> list[NominalPoint]: + """The nominal standing after each redemption, from the redemptions themselves. + + Everything the issuer repays per bond over its life is the original nominal, so the + nominal left after a given redemption is the sum of those still ahead of it. The final + entry is therefore zero on the maturity date, which is the truth: the paper is gone. + + A redemption with no money attached cannot be placed in the run and is warned about + rather than treated as zero — a silent zero would shift every later nominal upward. + """ + usable: list[tuple[date, Decimal, str | None]] = [] + for event in events: + if event.event_type != REDEMPTION: + warnings.append( + f"{event.instrument_uid}: незнакомый тип события облигации " + f"{event.event_type or '<пусто>'} — пропущено" + ) + continue + day = msk_date(event.pay_date) or msk_date(event.event_date) + if day is None: + continue + if event.pay_one_bond is None: + warnings.append(f"{event.instrument_uid}: погашение {day} без суммы — пропущено") + continue + usable.append((day, event.pay_one_bond, event.currency)) + + if not usable: + return [] + usable.sort() + remaining = sum((amount for _, amount, _ in usable), start=ZERO) + points: list[NominalPoint] = [] + for day, amount, ccy in usable: + remaining -= amount + points.append( + NominalPoint( + instrument_id=instrument_id, + effective_date=day, + nominal=remaining, + currency=(ccy or currency).upper(), + source=SOURCE, + ) + ) + return points + + +class TinvestEventsSource: + """`Source` protocol, same shape as `MoexSource`. Registration is done elsewhere.""" + + name = NAME + + async def sync(self, ctx: SyncContext) -> SyncResult: + token = ctx.settings.tinvest_token + if not token: + raise TinvestAuthError( + "TINVEST_TOKEN is not set — put a T-Invest token in .env " + "(t-bank.ru -> Инвестиции -> настройки -> токены)." + ) + + session = ctx.session + today = today_local() + targets = await load_targets(session) + if not targets: + log.info("tinvest_events: no T-Invest instruments in the ledger yet") + return SyncResult(cursor_after=today.isoformat(), counts={"payouts": 0}, changed=False) + + until = datetime(today.year + FORWARD_YEARS, 12, 31, tzinfo=UTC) + counts = {"instruments": 0, "dividends": 0, "coupons": 0, "redemptions": 0, "nominals": 0} + warnings: list[str] = [] + payouts: list[PayoutRow] = [] + nominals: list[NominalPoint] = [] + raw: list[dict[str, Any]] = [] + + async with TinvestClient(token) as client: + for target in targets: + counts["instruments"] += 1 + if target.asset_class in DIVIDEND_CLASSES: + rows = await client.dividends(target.uid, since=HISTORY_START, until=until) + counts["dividends"] += len(rows) + for row in rows: + payout = dividend_payout( + row, instrument_id=target.instrument_id, today=today + ) + if payout is None: + continue + payouts.append(payout) + raw.append( + _raw( + target.uid, + "dividend", + payout.source_id, + payout.record_date, + row.payload, + ) + ) + elif target.asset_class in BOND_CLASSES: + coupons = await client.bond_coupons( + target.uid, since=HISTORY_START, until=until + ) + counts["coupons"] += len(coupons) + for coupon in coupons: + payout = coupon_payout( + coupon, + instrument_id=target.instrument_id, + today=today, + warnings=warnings, + ) + if payout is None: + continue + payouts.append(payout) + raw.append( + _raw( + target.uid, + "coupon", + payout.source_id, + payout.pay_date, + coupon.payload, + ) + ) + + events = await client.bond_events( + target.uid, since=HISTORY_START, until=until, event_type=REDEMPTION + ) + counts["redemptions"] += len(events) + points = nominal_schedule( + events, + instrument_id=target.instrument_id, + currency=target.currency, + warnings=warnings, + ) + nominals += points + for index, event in enumerate(events): + day = msk_date(event.pay_date) or msk_date(event.event_date) + raw.append( + _raw( + target.uid, + "bond_event", + f"mty:{day if day else index}", + day, + event.payload, + ) + ) + + await _store_raw(session, raw) + written = await upsert_payouts(session, payouts) + counts["nominals"] = await upsert_nominals(session, nominals) + counts["payouts"] = written + await session.commit() + + log.info( + "tinvest_events: %s instruments, %s payouts, %s nominal points", + counts["instruments"], + counts["payouts"], + counts["nominals"], + ) + return SyncResult( + cursor_after=today.isoformat(), + counts=counts, + warnings=warnings, + changed=bool(written or counts["nominals"]), + ) + + +async def load_targets(session: AsyncSession) -> list[Target]: + """Every T-Invest instrument the confirmed ledger touches — held now or held once.""" + rows = ( + await session.execute( + select( + Instrument.id, + Instrument.tinvest_uid, + Instrument.asset_class, + Instrument.currency, + Instrument.ticker, + ) + .join(Event, Event.instrument_id == Instrument.id) + .where( + Event.status == EventStatus.confirmed, + Instrument.tinvest_uid.is_not(None), + Instrument.asset_class.in_(DIVIDEND_CLASSES | BOND_CLASSES), + ) + .group_by( + Instrument.id, + Instrument.tinvest_uid, + Instrument.asset_class, + Instrument.currency, + Instrument.ticker, + ) + ) + ).all() + return [ + Target( + instrument_id=r.id, + uid=r.tinvest_uid, + asset_class=r.asset_class, + currency=r.currency, + ticker=r.ticker, + ) + for r in rows + ] + + +def _raw( + uid: str, kind: str, source_id: str, day: date | None, payload: dict[str, Any] +) -> dict[str, Any]: + return { + "instrument_uid": uid, + "kind": kind, + "source_id": source_id, + "event_date": day, + "payload": payload, + } + + +async def _store_raw(session: AsyncSession, rows: Sequence[dict[str, Any]]) -> None: + """Append-only, idempotent on `(instrument_uid, kind, source_id)` — the raw-tier rule. + + The payload is refreshed rather than kept at its first version: these feeds revise a + schedule in place, and the point of the raw tier is to be able to re-derive the current + mapping, not to keep a history of what the API used to say. + """ + if not rows: + return + unique = {(r["instrument_uid"], r["kind"], r["source_id"]): r for r in rows} + stmt = pg_insert(RawTinvestEvent).values(list(unique.values())) + await session.execute( + stmt.on_conflict_do_update( + index_elements=["instrument_uid", "kind", "source_id"], + set_={ + "event_date": stmt.excluded.event_date, + "payload": stmt.excluded.payload, + "fetched_at": datetime.now(UTC), + }, + ) + ) + + +__all__ = [ + "KNOWN_COUPON_TYPES", + "NAME", + "REDEMPTION", + "Target", + "TinvestEventsSource", + "coupon_payout", + "dividend_payout", + "load_targets", + "msk_date", + "nominal_schedule", + "status_for", +] diff --git a/backend/tests/analytics/test_benchmarks.py b/backend/tests/analytics/test_benchmarks.py new file mode 100644 index 0000000..7a88907 --- /dev/null +++ b/backend/tests/analytics/test_benchmarks.py @@ -0,0 +1,227 @@ +"""Benchmarks on the portfolio's own grid — the acceptance check from the plan, фаза 4. + +«TWR и MCFTR на одной сетке без дыр в праздники»: the day the index has no quote must show +up in `days_skipped`, not quietly distort the return. And a price index must not be allowed +to pass as a total-return one — the two differ on identical holdings, and `kind` is what says +which is which. +""" + +from datetime import date, timedelta +from decimal import Decimal + +import pytest + +from factories import make_account, make_event, make_instrument, make_price, refresh +from fintracker.analytics.benchmarks import index_twr, opening_price, rebuild_benchmark_returns +from fintracker.api.schemas.benchmarks import BenchmarkReturnOut +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + Benchmark, + BenchmarkKind, + EventKind, + MetricBenchmarkReturns, +) + +D = Decimal +START = date(2025, 1, 1) + + +def day(n: int) -> date: + return START + timedelta(days=n) + + +# -------------------------------------------------------------------------------------- +# pure chain +# -------------------------------------------------------------------------------------- + + +def test_a_missing_quote_is_counted_not_smoothed_over(): + # the grid is every day; the index has no quote on day 2 (a holiday for it alone) + prices = {day(0): D(100), day(1): D(110), day(3): D(121)} + chain = index_twr(prices, [day(1), day(2), day(3)], opening=D(100)) + + assert chain.days_skipped == 1 + assert chain.days_used == 2 + # the move is not lost: day 3 links back to day 1's close, so the chain still telescopes + assert chain.value == D("0.210000") + + +def test_no_quote_at_all_gives_no_comparison_rather_than_zero(): + chain = index_twr({}, [day(1), day(2)], opening=None) + assert chain.value is None + assert chain.days_skipped == 2 + + +def test_the_period_may_open_on_a_day_the_index_did_not_trade(): + prices = {day(0): D(100), day(3): D(105)} + # day(1) is a Sunday for the index; the level it actually stood at is day(0)'s close + assert opening_price(prices, day(1)) == D(100) + assert opening_price(prices, day(-5)) is None + + +def test_kind_travels_all_the_way_out(): + # the client has to be able to mark a price-index comparison; the field is not optional + assert "kind" in BenchmarkReturnOut.model_fields + assert BenchmarkReturnOut.model_fields["kind"].annotation is str + + +# -------------------------------------------------------------------------------------- +# against a real portfolio +# -------------------------------------------------------------------------------------- + + +@pytest.fixture +async def portfolio(app) -> dict[str, object]: + """One share held for 40 days, priced every single day, so the grid has no holes.""" + from fintracker.analytics import today_local + + t = today_local() + bought = t - timedelta(days=40) + account = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + share = await make_instrument(ticker="GAZP", name="Газпром", asset_class=AssetClass.share) + await make_event(bought, account_id=account, kind=EventKind.deposit, amount="10000") + await make_event( + bought, + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="100", + price="100", + amount="-10000", + ) + for n in range(41): + await make_price(bought + timedelta(days=n), instrument_id=share, close=100 + n) + return {"account": account, "share": share, "bought": bought, "today": t} + + +async def _add_index( + code: str, kind: BenchmarkKind, closes: dict[date, str], *, ticker: str +) -> int: + instrument = await make_instrument( + ticker=ticker, name=code, asset_class=AssetClass.market_index, board="SNDX" + ) + for d, close in closes.items(): + await make_price(d, instrument_id=instrument, close=close) + async with get_sessionmaker()() as session: + benchmark = Benchmark( + code=code, + name=code, + kind=kind, + instrument_id=instrument, + source="moex", + currency="RUB", + is_default=kind is BenchmarkKind.total_return, + is_active=True, + ) + session.add(benchmark) + await session.commit() + await session.refresh(benchmark) + return benchmark.id + + +async def _rebuild_benchmarks() -> None: + async with get_sessionmaker()() as session: + await rebuild_benchmark_returns(session) + await session.commit() + + +async def _rows(scope: str = "all") -> dict[tuple[int, str], MetricBenchmarkReturns]: + from sqlalchemy import select + + async with get_sessionmaker()() as session: + found = await session.execute( + select(MetricBenchmarkReturns).where(MetricBenchmarkReturns.scope == scope) + ) + return {(r.benchmark_id, r.period): r for r in found.scalars()} + + +async def test_the_index_is_chained_over_the_portfolios_days_and_reports_the_holidays(portfolio): + """The plan's check: one grid, and a day the index misses is visible as a hole.""" + bought, today = portfolio["bought"], portfolio["today"] + holiday = bought + timedelta(days=20) + closes = { + bought + timedelta(days=n): str(1000 + n * 10) + for n in range(41) + if bought + timedelta(days=n) != holiday + } + benchmark = await _add_index("IMOEX", BenchmarkKind.price, closes, ticker="IMOEX") + + await refresh() + await _rebuild_benchmarks() + + rows = await _rows() + row = rows[(benchmark, "all")] + # the portfolio's own row defines the window; the benchmark copied it verbatim + from sqlalchemy import select + + from fintracker.models import MetricReturns + + async with get_sessionmaker()() as session: + found = await session.execute( + select(MetricReturns).where(MetricReturns.scope == "all", MetricReturns.period == "all") + ) + portfolio_row = found.scalar_one() + assert (row.date_from, row.date_to) == (portfolio_row.date_from, portfolio_row.date_to) + + # exactly one day of the compared window had no quote, and it is reported, not absorbed + assert row.days_skipped == 1 + assert row.twr is not None + # the chain still spans the whole window: 1000 -> 1400 over the priced days + assert row.twr == D("0.400000") + assert today >= portfolio_row.date_to + + +async def test_a_price_index_and_a_total_return_index_do_not_agree(portfolio): + """Same 40 days, same start: the dividend-bearing series ends higher, and says so.""" + bought = portfolio["bought"] + price_closes = {bought + timedelta(days=n): str(1000 + n * 10) for n in range(41)} + total_closes = {bought + timedelta(days=n): str(1000 + n * 15) for n in range(41)} + imoex = await _add_index("IMOEX", BenchmarkKind.price, price_closes, ticker="IMOEX") + mcftr = await _add_index("MCFTR", BenchmarkKind.total_return, total_closes, ticker="MCFTR") + + await refresh() + await _rebuild_benchmarks() + + rows = await _rows() + assert rows[(imoex, "all")].twr == D("0.400000") + assert rows[(mcftr, "all")].twr == D("0.600000") + # neither has a hole — the whole point of comparing against MCFTR rather than IMOEX is + # that the gap between them is dividends, not a difference in the days measured + assert rows[(imoex, "all")].days_skipped == 0 + assert rows[(mcftr, "all")].days_skipped == 0 + + +async def test_an_index_without_history_yields_no_number(portfolio): + """A benchmark nobody has quotes for is null, never 0 % — and it is reported.""" + from fintracker.analytics import FINDINGS + + benchmark = await _add_index("RGBITR", BenchmarkKind.total_return, {}, ticker="RGBITR") + await refresh() + FINDINGS.reset() + await _rebuild_benchmarks() + + rows = await _rows() + assert rows[(benchmark, "all")].twr is None + assert any(f.check_name == "benchmark_no_history" for f in FINDINGS.items) + + +async def test_nothing_in_the_metric_rows_is_a_float(portfolio): + bought = portfolio["bought"] + closes = {bought + timedelta(days=n): str(1000 + n * 10) for n in range(41)} + await _add_index("MCFTR", BenchmarkKind.total_return, closes, ticker="MCFTR") + await refresh() + await _rebuild_benchmarks() + + for row in (await _rows()).values(): + for value in (row.twr, row.twr_annualized): + assert value is None or isinstance(value, Decimal) diff --git a/backend/tests/analytics/test_goals.py b/backend/tests/analytics/test_goals.py new file mode 100644 index 0000000..ea4ead1 --- /dev/null +++ b/backend/tests/analytics/test_goals.py @@ -0,0 +1,342 @@ +"""Goal progress: the projection rules, then the refresh step end to end.""" + +from datetime import date, timedelta +from decimal import Decimal + +import pytest +from sqlalchemy import select + +from factories import make_account, make_event, make_instrument, make_price, refresh +from fintracker.analytics import today_local +from fintracker.analytics.goals import ( + MAX_HORIZON_MONTHS, + MIN_XIRR_HISTORY_DAYS, + evaluate, + monthly_needed, + months_between, + pick_rate, + rebuild_goal_progress, +) +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + EventKind, + Goal, + MetricGoalProgress, +) + +D = Decimal + + +def progress_of( + *, + current: str = "100000", + target: str = "200000", + target_date: date | None = None, + monthly: str | None = None, + xirr: str | None = None, + as_of: date | None = None, +): + return evaluate( + goal_id=1, + as_of=as_of or today_local(), + current=D(current), + target=D(target), + target_date=target_date, + monthly_contribution=None if monthly is None else D(monthly), + trailing_xirr=None if xirr is None else D(xirr), + ) + + +# --------------------------------------------------------------------------- the projection + + +def test_a_growing_portfolio_gets_a_date_in_the_future(): + result = progress_of(current="100000", target="200000", xirr="0.2") + assert result.basis == "xirr" + assert result.projected_date is not None + assert result.projected_date > today_local() + # 20 % a year doubles in a bit under four years + assert result.projected_date < today_local() + timedelta(days=365 * 5) + + +def test_a_flat_portfolio_with_no_contributions_gets_null_not_a_far_date(): + result = progress_of(current="100000", target="200000", xirr="0") + assert result.basis == "xirr" + assert result.projected_date is None + + +def test_a_falling_portfolio_with_no_contributions_gets_null(): + result = progress_of(current="100000", target="200000", xirr="-0.1") + assert result.basis == "xirr" + assert result.assumed_rate == D("-0.100000") + assert result.projected_date is None + + +def test_a_falling_portfolio_may_still_be_reached_by_contributions(): + result = progress_of(current="100000", target="200000", xirr="-0.02", monthly="20000") + assert result.basis == "xirr" + assert result.projected_date is not None + + +def test_without_a_trailing_return_the_plan_is_the_contributions(): + result = progress_of(current="100000", target="200000", monthly="10000") + assert result.basis == "contribution" + assert result.assumed_rate == D("0.000000") + # 100 000 left to raise at 10 000 a month is ten months of deposits + assert result.projected_date == _add(today_local(), 10) + + +def test_with_neither_a_return_nor_a_contribution_there_is_nothing_to_project(): + result = progress_of(current="100000", target="200000") + assert result.basis == "none" + assert result.projected_date is None + assert result.assumed_rate is None + + +def test_a_goal_already_met_is_projected_to_today(): + result = progress_of(current="300000", target="200000", xirr="0.1") + assert result.projected_date == today_local() + assert result.progress == D("1.500000") + + +def test_the_projection_gives_up_rather_than_naming_a_date_beyond_the_horizon(): + # 0.01 % a year against a target ten times away: reachable in theory, not in 30 years + result = progress_of(current="100000", target="1000000", xirr="0.0001") + assert result.projected_date is None + assert MAX_HORIZON_MONTHS == 360 + + +def _add(d: date, months: int) -> date: + from fintracker.analytics.goals import add_months + + return add_months(d, months) + + +# --------------------------------------------------------------------------- monthly needed + + +def test_monthly_needed_is_null_without_a_deadline(): + assert progress_of(monthly="1000").monthly_needed_rub is None + + +def test_monthly_needed_is_computed_when_there_is_a_deadline(): + as_of = date(2026, 1, 1) + result = progress_of( + current="100000", target="220000", target_date=date(2027, 1, 1), as_of=as_of + ) + # no growth assumed: 120 000 over 12 months + assert result.monthly_needed_rub == D("10000.00") + + +def test_monthly_needed_is_zero_when_the_trend_already_gets_there(): + as_of = date(2026, 1, 1) + result = progress_of( + current="100000", + target="105000", + target_date=date(2027, 1, 1), + xirr="0.2", + as_of=as_of, + ) + assert result.monthly_needed_rub == D(0) + assert result.on_track is True + + +def test_monthly_needed_is_null_once_the_deadline_has_passed(): + as_of = date(2026, 1, 1) + result = progress_of(target_date=date(2025, 1, 1), as_of=as_of, monthly="1000") + assert result.monthly_needed_rub is None + + +def test_a_deadline_the_trend_misses_is_not_on_track(): + as_of = date(2026, 1, 1) + result = progress_of( + current="100000", + target="200000", + target_date=date(2026, 6, 1), + monthly="1000", + as_of=as_of, + ) + assert result.on_track is False + + +def test_monthly_needed_accounts_for_the_assumed_growth(): + as_of = date(2026, 1, 1) + flat = progress_of(current="100000", target="220000", target_date=date(2027, 1, 1), as_of=as_of) + growing = progress_of( + current="100000", + target="220000", + target_date=date(2027, 1, 1), + xirr="0.2", + as_of=as_of, + ) + assert growing.monthly_needed_rub is not None + assert flat.monthly_needed_rub is not None + assert growing.monthly_needed_rub < flat.monthly_needed_rub + + +def test_months_between_counts_whole_months_only(): + assert months_between(date(2026, 1, 15), date(2027, 1, 14)) == 11 + assert months_between(date(2026, 1, 15), date(2027, 1, 15)) == 12 + assert months_between(date(2026, 5, 1), date(2026, 1, 1)) == 0 + + +# --------------------------------------------------------------------------- rate choice + + +def test_a_short_window_is_not_extrapolated_into_a_forecast(): + rows = [ + { + "period": "1m", + "date_from": date(2026, 8, 18), + "date_to": date(2026, 9, 18), + "xirr": D("3.5"), + } + ] + assert pick_rate(rows) == (None, "") + + +def test_the_shortest_qualifying_window_wins(): + rows = [ + { + "period": "all", + "date_from": date(2020, 1, 1), + "date_to": date(2026, 9, 18), + "xirr": D("0.05"), + }, + { + "period": "1y", + "date_from": date(2025, 9, 18), + "date_to": date(2026, 9, 18), + "xirr": D("0.18"), + }, + { + "period": "3m", + "date_from": date(2026, 6, 18), + "date_to": date(2026, 9, 18), + "xirr": D("9"), + }, + ] + assert pick_rate(rows) == (D("0.18"), "1y") + assert MIN_XIRR_HISTORY_DAYS == 180 + + +def test_a_period_without_an_xirr_is_skipped(): + rows = [ + { + "period": "1y", + "date_from": date(2025, 9, 18), + "date_to": date(2026, 9, 18), + "xirr": None, + } + ] + assert pick_rate(rows) == (None, "") + + +def test_every_number_in_the_progress_is_a_decimal(): + result = progress_of( + current="100000", target="200000", target_date=date(2030, 1, 1), xirr="0.1" + ) + for value in (result.current_value_rub, result.target_amount_rub, result.progress): + assert isinstance(value, Decimal) + assert isinstance(result.assumed_rate, Decimal) + assert isinstance(result.monthly_needed_rub, Decimal) + + +def test_monthly_needed_refuses_a_zero_month_window(): + assert monthly_needed(current=D(1), target=D(2), months=0, annual_rate=None) is None + + +# --------------------------------------------------------------------------- database + + +async def _portfolio(close_today: str) -> None: + """A year of history: 100 000 in, 1000 shares at 100, ending at `close_today`.""" + t = today_local() + start = t - timedelta(days=365) + account = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + share = await make_instrument(ticker="SBER", name="Сбербанк", asset_class=AssetClass.share) + await make_event(start, account_id=account, kind=EventKind.deposit, amount="100000") + await make_event( + start, + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="1000", + price="100", + amount="-100000", + ) + d = start + while d <= t: + close = "100" if d < t else close_today + await make_price(d, instrument_id=share, close=close) + d += timedelta(days=1) + await refresh() + + +async def _goal(**kwargs) -> int: + async with get_sessionmaker()() as session: + goal = Goal(**kwargs) + session.add(goal) + await session.commit() + await session.refresh(goal) + return goal.id + + +async def _rebuild() -> dict[int, MetricGoalProgress]: + async with get_sessionmaker()() as session: + await rebuild_goal_progress(session) + await session.commit() + rows = (await session.execute(select(MetricGoalProgress))).scalars().all() + return {r.goal_id: r for r in rows} + + +@pytest.mark.parametrize( + ("close_today", "reachable"), + [("150", True), ("100", False), ("60", False)], +) +async def test_the_projection_follows_the_real_trend(app, close_today: str, reachable: bool): + await _portfolio(close_today) + goal_id = await _goal(name="Капитал", scope="all", target_amount=D("1000000")) + rows = await _rebuild() + + row = rows[goal_id] + assert row.current_value_rub == D(close_today) * 1000 + assert row.progress == (D(close_today) * 1000 / D("1000000")).quantize(D("0.000001")) + assert row.basis == "xirr" + if reachable: + assert row.projected_date is not None and row.projected_date > today_local() + else: + assert row.projected_date is None + + +async def test_a_deadline_produces_a_monthly_need_and_none_without_one(app): + await _portfolio("100") + dated = await _goal( + name="С датой", + scope="all", + target_amount=D("400000"), + target_date=today_local() + timedelta(days=365), + ) + undated = await _goal(name="Без даты", scope="all", target_amount=D("400000")) + rows = await _rebuild() + + needed = rows[dated].monthly_needed_rub + assert needed is not None + assert needed > 0 + assert rows[undated].monthly_needed_rub is None + + +async def test_an_archived_goal_is_not_computed(app): + await _portfolio("100") + await _goal(name="Старое", scope="all", target_amount=D("1000"), archived=True) + assert await _rebuild() == {} diff --git a/backend/tests/analytics/test_income.py b/backend/tests/analytics/test_income.py new file mode 100644 index 0000000..16fef06 --- /dev/null +++ b/backend/tests/analytics/test_income.py @@ -0,0 +1,701 @@ +"""Income: the pure forecast rules first, then the rebuild end to end. + +The checks the plan names are here by name: every payout received in the last 12 months has a +`paid` calendar row, and a quarterly payer produces exactly four future entries carrying its +last amount. The third test is the one that catches real money: a coupon after an +amortisation, which must shrink with the nominal instead of staying at par. +""" + +from datetime import UTC, date, datetime, timedelta +from decimal import Decimal + +from sqlalchemy import select + +from factories import make_account, make_cbr_rate, make_event, make_instrument, refresh +from fintracker.analytics import FINDINGS, today_local +from fintracker.analytics.income import ( + BondFacts, + Entry, + Payment, + Payout, + add_months, + bond_entries, + coupon_per_unit, + detect_frequency, + drop_shadowed, + fold_payments, + history_entries, + monthly_rows, + nominal_at, + project_dates, + rebuild_income, + regular, + resolve_actions_fallback, +) +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + BondNominalSchedule, + CorporateAction, + CorporateActionKind, + CorporateActionStatus, + Event, + EventKind, + IncomeBasis, + Instrument, + MetricIncomeCalendar, + MetricIncomeMonthly, +) + +D = Decimal + + +# -------------------------------------------------------------------------------------- +# pure rules +# -------------------------------------------------------------------------------------- + + +def quarterly(n: int, *, end: date = date(2026, 9, 1)) -> list[date]: + return sorted(add_months(end, -3 * k) for k in range(n)) + + +def test_frequency_is_snapped_to_the_three_buckets_the_plan_allows(): + assert detect_frequency(quarterly(8)) == 4 + assert detect_frequency([date(2024, 6, 1), date(2024, 12, 1), date(2025, 6, 1)]) == 2 + assert detect_frequency([date(2024, 6, 1), date(2025, 6, 1), date(2026, 6, 1)]) == 1 + + +def test_a_single_payment_reads_as_annual_and_no_payment_reads_as_nothing(): + # one payment says nothing about spacing, but dropping a payout we have actually seen + # would hide it entirely — annual is the commonest Russian dividend + assert detect_frequency([date(2026, 5, 20)]) == 1 + assert detect_frequency([]) is None + + +def test_irregular_spacing_is_flagged_but_still_forecast(): + assert regular(quarterly(5)) + assert not regular([date(2025, 1, 10), date(2025, 2, 10), date(2026, 5, 10)]) + + +def test_a_quarterly_payer_gives_exactly_four_dates_in_a_year(): + last = date(2026, 6, 15) + dates = project_dates(last, 4, start=date(2026, 7, 1), end=date(2027, 6, 30)) + assert dates == [date(2026, 9, 15), date(2026, 12, 15), date(2027, 3, 15), date(2027, 6, 15)] + + +def test_a_coupon_follows_the_nominal_in_force_on_its_own_date(): + schedule = [(date(2024, 1, 1), D(1000)), (date(2026, 6, 1), D(500))] + assert nominal_at(schedule, date(2026, 5, 31)) == D(1000) + assert nominal_at(schedule, date(2026, 6, 1)) == D(500) + assert nominal_at(schedule, date(2023, 1, 1)) is None + # a coupon published against par halves once half the principal has been repaid + assert coupon_per_unit(D(40), D(1000), D(500)) == D(20) + assert coupon_per_unit(D(40), D(1000), D(1000)) == D(40) + assert coupon_per_unit(D(40), None, D(500)) == D(40) + + +def test_bond_entries_cover_coupon_amortisation_and_redemption(): + facts = BondFacts( + nominal=D(1000), + nominal_schedule=((date(2024, 1, 1), D(1000)), (date(2026, 11, 1), D(600))), + maturity_date=date(2027, 5, 1), + currency="RUB", + ) + coupons = [ + Payout(1, "coupon", "announced", None, date(2026, 10, 1), D(40), "RUB"), + Payout(1, "coupon", "announced", None, date(2027, 4, 1), D(40), "RUB"), + ] + entries = bond_entries(1, facts, coupons, D(10), start=date(2026, 9, 18), end=date(2027, 9, 18)) + by_kind = {(e.kind, e.expected_date): e for e in entries} + assert by_kind[("coupon", date(2026, 10, 1))].amount == D(400) + # after the amortisation the same published coupon is worth 60 % of itself + assert by_kind[("coupon", date(2027, 4, 1))].amount == D(240) + assert by_kind[("amortization", date(2026, 11, 1))].amount == D(4000) + assert by_kind[("repayment", date(2027, 5, 1))].amount == D(6000) + assert all(e.basis is IncomeBasis.schedule for e in entries) + + +def test_an_announced_payout_displaces_the_projection_of_the_same_payment(): + announced = [ + Entry( + 1, + "dividend", + date(2026, 10, 12), + date(2026, 10, 9), + D(20), + D(5), + D(100), + "RUB", + IncomeBasis.announced, + ) + ] + projected = [ + Entry( + 1, "dividend", date(2026, 10, 20), None, D(20), D(4), D(80), "RUB", IncomeBasis.history + ), + Entry( + 1, "dividend", date(2027, 4, 20), None, D(20), D(4), D(80), "RUB", IncomeBasis.history + ), + ] + kept = drop_shadowed(announced, projected) + # the declared autumn payment wins; the undeclared spring one survives + assert [e.expected_date for e in kept] == [date(2027, 4, 20)] + + +def test_the_fallback_resolver_prefers_the_strongest_status(): + same_day = date(2026, 10, 12) + payouts = [ + Payout(1, "dividend", "forecast", None, same_day, D(3), "RUB"), + Payout(1, "dividend", "announced", None, same_day, D(5), "RUB"), + Payout(1, "dividend", "cancelled", None, date(2026, 11, 1), D(9), "RUB"), + ] + resolved = resolve_actions_fallback(payouts) + assert [(p.status, p.amount_per_unit) for p in resolved] == [("announced", D(5))] + + +def payment(d: date, amount: str, *, held: str = "10", kind: str = "dividend") -> Payment: + return Payment( + account_id=1, + instrument_id=1, + kind=kind, + d=d, + currency="RUB", + amount=D(amount), + tax=D("0"), + held_qty=D(held), + ) + + +def test_history_extrapolates_the_last_amount_per_unit_onto_the_current_position(): + payments = [payment(d, "1000") for d in quarterly(8)] + entries, steady = history_entries( + 1, payments, D(5), start=date(2026, 9, 18), end=date(2027, 9, 17) + ) + assert steady + assert len(entries) == 4 + # 1000 ₽ on 10 units, now holding 5 — half the money, not the same money + assert {e.amount for e in entries} == {D(500)} + assert all(e.basis is IncomeBasis.history for e in entries) + + +def test_payments_fold_per_instrument_kind_and_day_across_accounts(): + d = date(2026, 8, 12) + entries = fold_payments([payment(d, "600", held="6"), payment(d, "400", held="4")]) + assert len(entries) == 1 + assert (entries[0].amount, entries[0].qty, entries[0].per_unit) == (D(1000), D(10), D(100)) + assert entries[0].basis is IncomeBasis.paid + + +def test_monthly_rows_group_by_month_kind_and_currency(): + rows = monthly_rows( + [ + payment(date(2026, 8, 3), "100"), + payment(date(2026, 8, 20), "200"), + payment(date(2026, 8, 20), "300", kind="coupon"), + payment(date(2026, 9, 1), "400"), + ] + ) + assert rows[(date(2026, 8, 1), "dividend", "RUB")] == (D(300), D(0), 2) + assert rows[(date(2026, 8, 1), "coupon", "RUB")] == (D(300), D(0), 1) + assert rows[(date(2026, 9, 1), "dividend", "RUB")] == (D(400), D(0), 1) + + +# -------------------------------------------------------------------------------------- +# the rebuild, against the database +# -------------------------------------------------------------------------------------- + + +async def broker_account() -> int: + return await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + + +async def make_bond( + *, + ticker: str = "RU000A0", + nominal: str = "1000", + maturity: date | None = None, + currency: str = "RUB", +) -> int: + async with get_sessionmaker()() as session: + bond = Instrument( + asset_class=AssetClass.bond, + ticker=ticker, + board="TQOB", + name=ticker, + currency=currency, + nominal=D(nominal), + nominal_currency=currency, + maturity_date=maturity, + ) + session.add(bond) + await session.commit() + await session.refresh(bond) + return bond.id + + +async def make_payout( + d: date, + *, + account_id: int, + instrument_id: int, + amount: str, + kind: EventKind = EventKind.dividend, + tax: str | None = None, + currency: str = "RUB", +) -> None: + """A received payout. `make_event` has no `tax`, and the tax column is the point here.""" + async with get_sessionmaker()() as session: + session.add( + Event( + account_id=account_id, + instrument_id=instrument_id, + kind=kind, + ts=datetime.combine(d, datetime.min.time(), tzinfo=UTC), + trade_date=d, + amount=D(amount), + currency=currency, + tax=D(tax) if tax is not None else None, + tax_currency=currency if tax is not None else None, + source="tinvest", + source_id=f"pay-{instrument_id}-{d}-{amount}-{kind}", + dedupe_key=f"tinvest:pay-{instrument_id}-{d}-{amount}-{kind}", + ) + ) + await session.commit() + + +async def make_action( + *, + instrument_id: int, + kind: CorporateActionKind, + status: CorporateActionStatus, + pay_date: date | None = None, + record_date: date | None = None, + amount_per_unit: str | None = None, + currency: str = "RUB", + source: str = "moex", +) -> None: + async with get_sessionmaker()() as session: + session.add( + CorporateAction( + instrument_id=instrument_id, + kind=kind, + status=status, + pay_date=pay_date, + record_date=record_date, + amount_per_unit=D(amount_per_unit) if amount_per_unit is not None else None, + currency=currency, + source=source, + source_id=f"{kind}-{pay_date}", + ) + ) + await session.commit() + + +async def make_nominal(instrument_id: int, effective: date, nominal: str) -> None: + async with get_sessionmaker()() as session: + session.add( + BondNominalSchedule( + instrument_id=instrument_id, + effective_date=effective, + nominal=D(nominal), + currency="RUB", + source="moex", + ) + ) + await session.commit() + + +async def rebuild() -> None: + async with get_sessionmaker()() as session: + await rebuild_income(session) + await session.commit() + + +async def calendar(scope: str = "all") -> list[MetricIncomeCalendar]: + async with get_sessionmaker()() as session: + return list( + ( + await session.execute( + select(MetricIncomeCalendar) + .where(MetricIncomeCalendar.scope == scope) + .order_by(MetricIncomeCalendar.expected_date) + ) + ) + .scalars() + .all() + ) + + +async def monthly(scope: str = "all") -> list[MetricIncomeMonthly]: + async with get_sessionmaker()() as session: + return list( + ( + await session.execute( + select(MetricIncomeMonthly) + .where(MetricIncomeMonthly.scope == scope) + .order_by(MetricIncomeMonthly.month, MetricIncomeMonthly.kind) + ) + ) + .scalars() + .all() + ) + + +def forecast_within(rows, months: int = 12) -> list[MetricIncomeCalendar]: + """Future rows inside a half-open window of `months`, the way the API reads them.""" + today = today_local() + end = add_months(today, months) + return [r for r in rows if r.basis is not IncomeBasis.paid and today <= r.expected_date < end] + + +async def test_every_payout_of_the_last_year_has_a_paid_calendar_row(app): + """Plan check: каждый полученный дивиденд/купон за 12 мес имеет запись календаря.""" + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="SBER", name="Сбербанк") + bond = await make_bond() + await make_event( + today - timedelta(days=400), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="20", + price="250", + amount="-5000", + ) + await make_event( + today - timedelta(days=400), + account_id=account, + kind=EventKind.buy, + instrument_id=bond, + quantity="10", + price="1000", + amount="-10000", + ) + paid_days = [30, 120, 210, 300] + for offset in paid_days: + await make_payout( + today - timedelta(days=offset), + account_id=account, + instrument_id=share, + amount="400", + ) + await make_payout( + today - timedelta(days=60), + account_id=account, + instrument_id=bond, + amount="400", + kind=EventKind.coupon, + ) + await refresh() + await rebuild() + + rows = await calendar() + paid = {(r.instrument_id, r.kind, r.expected_date) for r in rows if r.basis is IncomeBasis.paid} + for offset in paid_days: + assert (share, "dividend", today - timedelta(days=offset)) in paid + assert (bond, "coupon", today - timedelta(days=60)) in paid + assert len(paid) == len(paid_days) + 1 + + +async def test_a_quarterly_payer_gives_four_future_entries_with_the_last_amount(app): + """Plan check: квартальный плательщик даёт 4 будущих записи с последней суммой.""" + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="LKOH", name="Лукойл") + await make_event( + add_months(today, -30), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="100", + amount="-1000", + ) + for k in range(1, 9): # eight payments, one a quarter, the last three months ago + await make_payout( + add_months(today, -3 * k), + account_id=account, + instrument_id=share, + amount="500", + ) + await refresh() + await rebuild() + + future = forecast_within(await calendar()) + assert len(future) == 4 + assert {r.basis for r in future} == {IncomeBasis.history} + assert {r.amount for r in future} == {D(500)} + assert {r.per_unit for r in future} == {D(50)} + + +async def test_a_coupon_shrinks_with_the_nominal_after_an_amortisation(app): + today = today_local() + account = await broker_account() + bond = await make_bond(ticker="RU000AMORT") + await make_event( + today - timedelta(days=30), + account_id=account, + kind=EventKind.buy, + instrument_id=bond, + quantity="10", + price="1000", + amount="-10000", + ) + await make_nominal(bond, today - timedelta(days=700), "1000") + await make_nominal(bond, add_months(today, 2), "500") + for months in (1, 4): + await make_action( + instrument_id=bond, + kind=CorporateActionKind.coupon, + status=CorporateActionStatus.announced, + pay_date=add_months(today, months), + amount_per_unit="40", + ) + await refresh() + await rebuild() + + coupons = {r.expected_date: r for r in await calendar() if r.kind == "coupon"} + before = coupons[add_months(today, 1)] + after = coupons[add_months(today, 4)] + assert before.basis is IncomeBasis.schedule + assert (before.per_unit, before.amount) == (D(40), D(400)) + # half the principal has been repaid, so the same published coupon pays half + assert (after.per_unit, after.amount) == (D(20), D(200)) + # ...and the amortisation itself is a payment, priced off the step in the schedule + amortisation = next(r for r in await calendar() if r.kind == "amortization") + assert (amortisation.expected_date, amortisation.amount) == (add_months(today, 2), D(5000)) + + +async def test_a_sold_position_leaves_the_forecast_and_a_halved_one_halves_it(app): + today = today_local() + account = await broker_account() + kept = await make_instrument(ticker="GAZP", name="Газпром") + gone = await make_instrument(ticker="MGNT", name="Магнит") + for instrument in (kept, gone): + await make_event( + add_months(today, -18), + account_id=account, + kind=EventKind.buy, + instrument_id=instrument, + quantity="20", + price="100", + amount="-2000", + ) + await make_payout( + add_months(today, -12), + account_id=account, + instrument_id=instrument, + amount="2000", + ) + await make_event( + add_months(today, -2), + account_id=account, + kind=EventKind.sell, + instrument_id=gone, + quantity="-20", + price="100", + amount="2000", + ) + await make_event( + add_months(today, -2), + account_id=account, + kind=EventKind.sell, + instrument_id=kept, + quantity="-10", + price="100", + amount="1000", + ) + await refresh() + await rebuild() + + future = forecast_within(await calendar()) + assert [r.instrument_id for r in future] == [kept] + # 2000 ₽ on 20 units, 10 units left: the forecast follows the position, not the history + assert future[0].amount == D(1000) + + +async def test_an_announced_dividend_beats_the_history_of_the_same_payment(app): + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="TATN", name="Татнефть") + await make_event( + add_months(today, -18), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="500", + amount="-5000", + ) + await make_payout(add_months(today, -12), account_id=account, instrument_id=share, amount="300") + announced_on = today + timedelta(days=10) + await make_action( + instrument_id=share, + kind=CorporateActionKind.dividend, + status=CorporateActionStatus.announced, + record_date=today + timedelta(days=7), + pay_date=announced_on, + amount_per_unit="45", + ) + await refresh() + await rebuild() + + future = forecast_within(await calendar()) + assert len(future) == 1 + row = future[0] + assert (row.basis, row.expected_date, row.amount) == ( + IncomeBasis.announced, + announced_on, + D(450), + ) + assert row.record_date == today + timedelta(days=7) + + +async def test_history_groups_by_month_kind_and_currency_and_sums_the_tax(app): + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="PHOR", name="ФосАгро") + bond = await make_bond(ticker="RU000TAX") + await make_event( + add_months(today, -12), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="100", + amount="-1000", + ) + month = add_months(today.replace(day=5), -3) + await make_payout(month, account_id=account, instrument_id=share, amount="870", tax="130") + await make_payout( + month + timedelta(days=10), + account_id=account, + instrument_id=share, + amount="435", + tax="65", + ) + await make_payout( + month + timedelta(days=2), + account_id=account, + instrument_id=bond, + amount="400", + kind=EventKind.coupon, + ) + await refresh() + await rebuild() + + rows = {(r.month, r.kind): r for r in await monthly()} + dividends = rows[(month.replace(day=1), "dividend")] + assert (dividends.amount, dividends.tax_withheld, dividends.payment_count) == ( + D(1305), + D(195), + 2, + ) + assert dividends.currency == "RUB" + assert rows[(month.replace(day=1), "coupon")].amount == D(400) + + +async def test_a_payment_without_a_rate_keeps_its_row_and_loses_only_the_rouble_column(app): + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="TCS", name="TCS Group", currency="USD") + await make_event( + add_months(today, -12), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="10", + amount="-100", + currency="USD", + ) + paid_on = add_months(today, -2) + await make_payout(paid_on, account_id=account, instrument_id=share, amount="20", currency="USD") + await refresh() + FINDINGS.reset() + await rebuild() + + row = next(r for r in await calendar() if r.basis is IncomeBasis.paid) + assert (row.amount, row.currency, row.amount_rub) == (D(20), "USD", None) + assert next(r for r in await monthly()).amount_rub is None + assert any(f.check_name == "income_missing_fx" for f in FINDINGS.items) + + +async def test_a_rate_on_the_payment_date_fills_the_rouble_column(app): + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="TCS", name="TCS Group", currency="USD") + await make_event( + add_months(today, -12), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="10", + amount="-100", + currency="USD", + ) + paid_on = add_months(today, -2) + await make_cbr_rate(paid_on, "USD", "90") + await make_payout(paid_on, account_id=account, instrument_id=share, amount="20", currency="USD") + await refresh() + await rebuild() + + row = next(r for r in await calendar() if r.basis is IncomeBasis.paid) + assert row.amount_rub == D(1800) + + +async def test_an_instrument_with_neither_schedule_nor_history_is_a_warning_not_a_zero(app): + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="SILENT", name="Ничего не платит") + await make_event( + add_months(today, -6), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="100", + amount="-1000", + ) + await refresh() + FINDINGS.reset() + await rebuild() + + assert forecast_within(await calendar()) == [] + finding = next(f for f in FINDINGS.items if f.check_name == "income_without_history") + assert finding.severity == "warn" + assert finding.ref == {"instruments": [share]} + + +async def test_the_tables_are_rebuilt_from_scratch_on_every_run(app): + today = today_local() + account = await broker_account() + share = await make_instrument(ticker="ROSN", name="Роснефть") + await make_event( + add_months(today, -12), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="10", + price="100", + amount="-1000", + ) + await make_payout(add_months(today, -6), account_id=account, instrument_id=share, amount="500") + await refresh() + await rebuild() + before = len(await calendar()) + await rebuild() + assert len(await calendar()) == before + assert before > 0 diff --git a/backend/tests/analytics/test_rebalance.py b/backend/tests/analytics/test_rebalance.py new file mode 100644 index 0000000..69c7ea6 --- /dev/null +++ b/backend/tests/analytics/test_rebalance.py @@ -0,0 +1,477 @@ +"""Rebalancing: the pure planning rules, then the refresh step end to end.""" + +from datetime import timedelta +from decimal import Decimal + +import pytest +from sqlalchemy import select + +from factories import make_account, make_event, make_instrument, make_price, refresh +from fintracker.analytics import FINDINGS, today_local +from fintracker.analytics.rebalance import ( + Position, + Target, + build_plan, + compute_rebalance, + rebuild_rebalance, +) +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AllocationDimension, + AssetClass, + EventKind, + Instrument, + MetricAllocation, + MetricRebalance, + Portfolio, + PortfolioAccount, + PortfolioTarget, +) + +D = Decimal +DIM = AllocationDimension + + +def position( + instrument_id: int = 1, + *, + ticker: str = "SBER", + lot: int = 10, + qty: str = "100", + unit: str = "275", +) -> Position: + return Position( + instrument_id=instrument_id, + ticker=ticker, + name=ticker, + lot=lot, + qty=D(qty), + unit_value_rub=D(unit), + price=D(unit), + price_currency="RUB", + ) + + +def plan( + *, + bucket_values: dict[str, Decimal], + positions: dict[str, list[Position]], + targets: dict[str, Target], + cash: str = "1000000", +): + total = sum((v for v in bucket_values.values() if v > 0), start=D(0)) + return build_plan( + portfolio_id=1, + dimension=DIM.asset_class, + as_of=today_local(), + total_value_rub=total, + bucket_values=bucket_values, + positions=positions, + targets=targets, + cash_available_rub=D(cash), + ) + + +def bucket(result, name: str): + return next(b for b in result.buckets if b.bucket == name) + + +# --------------------------------------------------------------------------- lots and cash + + +def test_a_buy_is_whole_lots_even_when_the_money_would_stretch_further(): + # 100 lots' worth of money, a lot of 10 at 275 => 2750 a lot + result = plan( + bucket_values={"share": D("27500"), "cash": D("22500")}, + positions={"share": [position(qty="100", unit="275", lot=10)]}, + targets={"share": Target(D("0.8")), "cash": Target(D("0.2"))}, + cash="22500", + ) + trade = bucket(result, "share").trades[0] + assert trade.action == "buy" + # 0.8 * 50000 - 27500 = 12500 -> 45.45 units -> 4 lots = 40, never 45 + assert trade.qty == D(40) + assert trade.qty % trade.lot == 0 + assert trade.amount_rub == D(40) * D("275") + + +def test_a_buy_is_cut_to_the_cash_on_hand_and_says_so(): + result = plan( + bucket_values={"share": D("27500"), "cash": D("22500")}, + positions={"share": [position(qty="100", unit="275", lot=10)]}, + targets={"share": Target(D("0.8")), "cash": Target(D("0.2"))}, + cash="6000", + ) + trade = bucket(result, "share").trades[0] + # 6000 buys two lots (5500), not the 4 the target asks for + assert trade.qty == D(20) + assert trade.blocked_by_cash is True + assert trade.amount_rub <= D("6000") + + +def test_no_cash_at_all_still_reports_the_blocked_buy_rather_than_hiding_it(): + result = plan( + bucket_values={"share": D("27500"), "cash": D("22500")}, + positions={"share": [position(qty="100", unit="275", lot=10)]}, + targets={"share": Target(D("0.8")), "cash": Target(D("0.2"))}, + cash="0", + ) + trade = bucket(result, "share").trades[0] + assert trade.qty == D(0) + assert trade.blocked_by_cash is True + + +def test_cash_is_spent_once_across_buckets(): + result = plan( + bucket_values={"share": D("1000"), "bond": D("1000"), "cash": D("8000")}, + positions={ + "share": [position(1, ticker="SBER", qty="10", unit="100", lot=1)], + "bond": [position(2, ticker="OFZ", qty="10", unit="100", lot=1)], + }, + targets={"share": Target(D("0.45")), "bond": Target(D("0.45")), "cash": Target(D("0.1"))}, + cash="1000", + ) + spent = sum(t.amount_rub for b in result.buckets for t in b.trades if t.action == "buy") + assert spent <= D("1000") + + +# --------------------------------------------------------------------------- the band + + +def test_a_drift_inside_the_band_proposes_nothing(): + result = plan( + bucket_values={"share": D("6200"), "bond": D("3800")}, + positions={"share": [position(qty="62", unit="100", lot=1)]}, + targets={"share": Target(D("0.6"), D("0.05")), "bond": Target(D("0.4"), D("0.05"))}, + ) + share = bucket(result, "share") + assert share.drift == D("0.02") + assert share.within_band is True + assert share.trades == [] + assert share.delta_value_rub == D(0) + + +def test_the_same_drift_outside_the_band_proposes_a_trade(): + result = plan( + bucket_values={"share": D("6200"), "bond": D("3800")}, + positions={ + "share": [position(qty="62", unit="100", lot=1)], + "bond": [position(2, ticker="OFZ", qty="38", unit="100", lot=1)], + }, + targets={"share": Target(D("0.6"), D("0.01")), "bond": Target(D("0.4"), D("0.01"))}, + ) + share = bucket(result, "share") + assert share.within_band is False + assert share.trades[0].action == "sell" + assert share.trades[0].qty == D(2) + + +# --------------------------------------------------------------------------- sells + + +def test_a_sell_never_exceeds_the_position_and_never_goes_short(): + # the bucket must shrink by more than it holds: the target moved to zero + result = plan( + bucket_values={"share": D("1000"), "bond": D("9000")}, + positions={"share": [position(qty="10", unit="100", lot=1)]}, + targets={"share": Target(D("0")), "bond": Target(D("1"))}, + ) + trade = bucket(result, "share").trades[0] + assert trade.action == "sell" + assert trade.qty == D(10) + assert trade.qty <= D(10) + + +def test_a_sell_is_capped_to_whole_lots_of_what_is_held(): + # 25 units of a 10-lot paper: at most two lots can be sold + result = plan( + bucket_values={"share": D("2500"), "bond": D("7500")}, + positions={"share": [position(qty="25", unit="100", lot=10)]}, + targets={"share": Target(D("0")), "bond": Target(D("1"))}, + ) + trade = bucket(result, "share").trades[0] + assert trade.qty == D(20) + + +def test_a_bucket_is_trimmed_proportionally_not_from_one_paper(): + result = plan( + bucket_values={"share": D("10000"), "bond": D("0")}, + positions={ + "share": [ + position(1, ticker="BIG", qty="75", unit="100", lot=1), + position(2, ticker="SMALL", qty="25", unit="100", lot=1), + ] + }, + targets={"share": Target(D("0.5")), "bond": Target(D("0.5"))}, + ) + by_ticker = {t.ticker: t.qty for t in bucket(result, "share").trades} + # 5000 to raise, split 75/25 by value: 37 and 12 units (floored to whole lots) + assert by_ticker == {"BIG": D(37), "SMALL": D(12)} + + +def test_a_bucket_with_nothing_priced_in_it_warns_instead_of_inventing_a_trade(): + result = plan( + bucket_values={"share": D("10000"), "bond": D("0")}, + positions={}, + targets={"share": Target(D("0.5")), "bond": Target(D("0.5"))}, + ) + assert bucket(result, "share").trades == [] + assert any("share" in w for w in result.warnings) + + +def test_the_cash_bucket_needs_no_trades_and_produces_no_warning(): + result = plan( + bucket_values={"share": D("5000"), "cash": D("5000")}, + positions={"share": [position(qty="50", unit="100", lot=1)]}, + targets={"share": Target(D("0.9")), "cash": Target(D("0.1"))}, + cash="5000", + ) + assert bucket(result, "cash").trades == [] + assert not any("cash" in w for w in result.warnings) + + +def test_a_bucket_without_a_target_is_reported_but_never_traded(): + result = plan( + bucket_values={"share": D("5000"), "etf": D("5000")}, + positions={"etf": [position(2, ticker="TMOS", qty="50", unit="100", lot=1)]}, + targets={"share": Target(D("1"))}, + ) + etf = bucket(result, "etf") + assert etf.target_weight is None + assert etf.drift is None + assert etf.trades == [] + + +def test_every_number_in_the_plan_is_a_decimal(): + result = plan( + bucket_values={"share": D("6200"), "bond": D("3800")}, + positions={"share": [position(qty="62", unit="100", lot=1)]}, + targets={"share": Target(D("0.5")), "bond": Target(D("0.5"))}, + ) + for b in result.buckets: + for value in (b.current_value_rub, b.current_weight, b.delta_value_rub): + assert isinstance(value, Decimal) + for t in b.trades: + for value in (t.qty, t.price, t.amount_rub): + assert isinstance(value, Decimal) + + +# --------------------------------------------------------------------------- database + + +async def _portfolio_with(*, unpriced: bool) -> dict[str, int]: + """A broker account in a portfolio: 500 SBER (lot 10), 20 OFZ, the rest in cash.""" + t = today_local() + bought = t - timedelta(days=40) + account = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + sber = await make_instrument(ticker="SBER", name="Сбербанк", asset_class=AssetClass.share) + ofz = await make_instrument(ticker="OFZ", name="ОФЗ", asset_class=AssetClass.bond) + + async with get_sessionmaker()() as session: + instrument = await session.get(Instrument, sber) + assert instrument is not None + instrument.lot = 10 + portfolio = Portfolio(name="Основной") + session.add(portfolio) + await session.flush() + session.add(PortfolioAccount(portfolio_id=portfolio.id, account_id=account)) + portfolio_id = portfolio.id + await session.commit() + + await make_event(bought, account_id=account, kind=EventKind.deposit, amount="100000") + await make_event( + bought, + account_id=account, + kind=EventKind.buy, + instrument_id=sber, + quantity="500", + price="100", + amount="-50000", + ) + await make_event( + bought, + account_id=account, + kind=EventKind.buy, + instrument_id=ofz, + quantity="20", + price="1000", + amount="-20000", + ) + ids = {"account": account, "portfolio": portfolio_id, "sber": sber, "ofz": ofz} + + if unpriced: + silent = await make_instrument( + ticker="SIBN6P4", name="Без цены", asset_class=AssetClass.share, board="SPBRUBND" + ) + await make_event( + bought, + account_id=account, + kind=EventKind.buy, + instrument_id=silent, + quantity="5", + price="1000", + amount="-5000", + ) + ids["silent"] = silent + + d = bought + while d <= t: + await make_price(d, instrument_id=sber, close="100") + await make_price(d, instrument_id=ofz, close="1000") + d += timedelta(days=1) + await refresh() + return ids + + +async def _set_targets(portfolio_id: int, rows: list[tuple[str, str, str]]) -> None: + async with get_sessionmaker()() as session: + for bucket_name, weight, band in rows: + session.add( + PortfolioTarget( + portfolio_id=portfolio_id, + dimension=DIM.asset_class, + bucket=bucket_name, + target_weight=D(weight), + band=D(band), + ) + ) + await session.commit() + + +@pytest.fixture +async def portfolio(app) -> dict[str, int]: + ids = await _portfolio_with(unpriced=False) + await _set_targets( + ids["portfolio"], + [("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")], + ) + return ids + + +async def test_the_step_fills_the_target_columns_of_metric_allocation(portfolio): + async with get_sessionmaker()() as session: + await rebuild_rebalance(session) + await session.commit() + rows = ( + ( + await session.execute( + select(MetricAllocation).where( + MetricAllocation.scope == f"portfolio:{portfolio['portfolio']}", + MetricAllocation.dimension == DIM.asset_class, + ) + ) + ) + .scalars() + .all() + ) + by_bucket = {r.bucket: r for r in rows} + assert by_bucket["share"].target_weight == D("0.6") + assert by_bucket["share"].weight == D("0.5") + assert by_bucket["share"].drift == D("-0.1") + assert by_bucket["cash"].target_weight == D("0.2") + assert by_bucket["cash"].drift == D("0.1") + + +async def test_metric_rebalance_agrees_with_metric_allocation(portfolio): + async with get_sessionmaker()() as session: + await rebuild_rebalance(session) + await session.commit() + allocation = { + r.bucket: r + for r in ( + ( + await session.execute( + select(MetricAllocation).where( + MetricAllocation.scope == f"portfolio:{portfolio['portfolio']}", + MetricAllocation.dimension == DIM.asset_class, + ) + ) + ) + .scalars() + .all() + ) + } + summaries = { + r.bucket: r + for r in ( + ( + await session.execute( + select(MetricRebalance).where(MetricRebalance.instrument_id.is_(None)) + ) + ) + .scalars() + .all() + ) + } + trades = ( + ( + await session.execute( + select(MetricRebalance).where(MetricRebalance.instrument_id.is_not(None)) + ) + ) + .scalars() + .all() + ) + + for name, row in summaries.items(): + assert row.current_weight == allocation[name].weight + assert row.target_weight == allocation[name].target_weight + assert row.current_value_rub == allocation[name].value_rub + + # 0.6 of 100 000 is 60 000 against 50 000 held: 100 more shares at 100, lot 10 + buy = next(t for t in trades if t.instrument_id == portfolio["sber"]) + assert buy.suggested_qty == D(100) + assert buy.suggested_qty is not None + assert buy.lot is not None + assert buy.suggested_qty % buy.lot == 0 + assert buy.blocked_by_cash is False + # the bond bucket sits exactly on its target and proposes nothing + assert summaries["bond"].within_band is True + assert not [t for t in trades if t.instrument_id == portfolio["ofz"]] + + +async def test_an_instrument_without_a_price_is_left_out_but_reported(app): + ids = await _portfolio_with(unpriced=True) + await _set_targets( + ids["portfolio"], + [("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")], + ) + FINDINGS.reset() + async with get_sessionmaker()() as session: + await rebuild_rebalance(session) + await session.commit() + trades = ( + ( + await session.execute( + select(MetricRebalance).where(MetricRebalance.instrument_id.is_not(None)) + ) + ) + .scalars() + .all() + ) + assert ids["silent"] not in {t.instrument_id for t in trades} + assert any( + f.check_name == "rebalance_incomplete" and "SIBN6P4" in f.detail for f in FINDINGS.items + ) + + +async def test_the_what_if_cash_overrides_the_real_balance(portfolio): + async with get_sessionmaker()() as session: + real = await compute_rebalance(session, portfolio["portfolio"], DIM.asset_class) + poor = await compute_rebalance( + session, portfolio["portfolio"], DIM.asset_class, cash_available_rub=D("500") + ) + assert real.cash_available_rub == D("30000") + rich_trade = next(t for b in real.buckets for t in b.trades) + poor_trade = next(t for b in poor.buckets for t in b.trades) + assert poor_trade.qty < rich_trade.qty + assert poor_trade.blocked_by_cash is True diff --git a/backend/tests/analytics/test_tax.py b/backend/tests/analytics/test_tax.py new file mode 100644 index 0000000..db12bfa --- /dev/null +++ b/backend/tests/analytics/test_tax.py @@ -0,0 +1,367 @@ +"""The tax year, checked against an example worked out by hand — the plan's фаза-4 check. + +Everything here is an estimate by construction (the broker is the tax agent), so the tests +are about the two things that make the estimate worth having: that it is reproducible on +paper, and that it never invents a number it does not have. +""" + +from datetime import date +from decimal import Decimal + +import pytest +from sqlalchemy import select + +from factories import ( + make_account, + make_cbr_rate, + make_event, + make_instrument, +) +from fintracker.analytics import FINDINGS +from fintracker.analytics.tax import TAX_RATE, rebuild_tax_year +from fintracker.db import get_sessionmaker +from fintracker.ledger.rebuild import rebuild_lots +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + EventKind, + LotDisposal, + MetricTaxYear, +) +from fintracker.pricing.fx import rebuild_fx_daily + +D = Decimal + + +async def _rebuild() -> None: + """The three steps a tax year depends on, without the rest of the refresh.""" + FINDINGS.reset() + async with get_sessionmaker()() as session: + await rebuild_fx_daily(session) + await session.commit() + await rebuild_lots(session) + await session.commit() + await rebuild_tax_year(session) + await session.commit() + + +async def _rows() -> dict[tuple[int, int], MetricTaxYear]: + async with get_sessionmaker()() as session: + found = await session.execute(select(MetricTaxYear)) + return {(r.year, r.account_id): r for r in found.scalars()} + + +async def _payment( + account_id: int, + instrument_id: int, + kind: EventKind, + d: date, + amount: str, + *, + tax: str | None = None, + currency: str = "RUB", +) -> None: + """A dividend or coupon as a broker reports it: net cash plus the tax it kept back.""" + from datetime import UTC, datetime, time + + from fintracker.models import Event + + key = f"{kind}-{instrument_id}-{d}" + async with get_sessionmaker()() as session: + session.add( + Event( + account_id=account_id, + instrument_id=instrument_id, + kind=kind, + ts=datetime.combine(d, time.min, tzinfo=UTC), + trade_date=d, + amount=D(amount), + currency=currency, + tax=None if tax is None else D(tax), + tax_currency=None if tax is None else currency, + source="tinvest", + source_id=key, + dedupe_key=f"tinvest:{key}", + ) + ) + await session.commit() + + +async def _broker_account(name: str = "Брокерский") -> int: + return await make_account( + name=name, + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + + +@pytest.fixture +async def usd_rates(app) -> None: + """Two CBR quotes: the dollar rose by 20 % between the purchase and the sale.""" + await make_cbr_rate(date(2023, 3, 10), "USD", "75") + await make_cbr_rate(date(2024, 6, 20), "USD", "90") + + +async def test_the_worked_example_adds_up(usd_rates): + """Done on paper first; the code has to agree with the paper, not the other way round. + + Purchase 10.03.2023: 10 units at $100 = $1000, CBR 75 ₽/$ -> cost 75 000 ₽ + Sale 20.06.2024: 10 units at $110 = $1100, CBR 90 ₽/$ -> proceeds 99 000 ₽ + Realised in roubles = 24 000 ₽ + of which the price move is $100 x 90 = 9 000 ₽ + and currency revaluation $1000 x 15 = 15 000 ₽ (in the base, plan §7 q4) + + Plus a rouble lot bought 10.01.2020 and sold the same day in 2024 for +1 000 ₽. It is + held over three years, so art. 219.1 takes its result back out of the base. + + gain 24 000 + 1 000 = 25 000 ₽ + loss 0 + ЛДВ exempt 1 000 ₽ + base 25 000 - 1 000 = 24 000 ₽ + tax 24 000 x 0.13 = 3 120 ₽ + """ + account = await _broker_account() + foreign = await make_instrument(ticker="AAPL", name="Apple", currency="USD") + old = await make_instrument(ticker="SBER", name="Сбербанк") + + await make_event( + date(2023, 3, 10), + account_id=account, + kind=EventKind.buy, + instrument_id=foreign, + quantity="10", + price="100", + amount="-1000", + currency="USD", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=foreign, + quantity="-10", + price="110", + amount="1100", + currency="USD", + ) + await make_event( + date(2020, 1, 10), + account_id=account, + kind=EventKind.buy, + instrument_id=old, + quantity="10", + price="100", + amount="-1000", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=old, + quantity="-10", + price="200", + amount="2000", + ) + + await _rebuild() + + row = (await _rows())[(2024, account)] + assert row.realized_gain_rub == D("25000.00") + assert row.realized_loss_rub == D("0.00") + assert row.ldv_exempt_rub == D("1000.00") + assert row.taxable_base_rub == D("24000.00") + assert row.estimated_tax_rub == D("3120.00") + # the rate is recorded, not implied, so a future change stays visible in old years + assert row.tax_rate == TAX_RATE == D("0.13") + + +async def test_the_three_year_lot_is_flagged_by_the_ledgers_own_rule(usd_rates): + """`ldv_eligible` is computed once, in `ledger/lots.py`; the tax view only reads it.""" + account = await _broker_account() + instrument = await make_instrument(ticker="SBER", name="Сбербанк") + await make_event( + date(2020, 1, 10), + account_id=account, + kind=EventKind.buy, + instrument_id=instrument, + quantity="10", + price="100", + amount="-1000", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=instrument, + quantity="-10", + price="200", + amount="2000", + ) + await _rebuild() + + async with get_sessionmaker()() as session: + disposal = (await session.execute(select(LotDisposal))).scalar_one() + assert disposal.holding_days >= 3 * 365 + assert disposal.ldv_eligible is True + + row = (await _rows())[(2024, account)] + assert row.ldv_exempt_rub == D("1000.00") + assert row.taxable_base_rub == D("0.00") + assert row.estimated_tax_rub == D("0.00") + + +async def test_currency_revaluation_is_not_the_currency_result(usd_rates): + """A position flat in dollars still owes tax when the dollar rose — and the two numbers + must not be confused: the base is 15 000 ₽ while the dollar result is exactly zero.""" + account = await _broker_account() + instrument = await make_instrument(ticker="AAPL", name="Apple", currency="USD") + await make_event( + date(2023, 3, 10), + account_id=account, + kind=EventKind.buy, + instrument_id=instrument, + quantity="10", + price="100", + amount="-1000", + currency="USD", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=instrument, + quantity="-10", + price="100", + amount="1000", + currency="USD", + ) + await _rebuild() + + async with get_sessionmaker()() as session: + disposal = (await session.execute(select(LotDisposal))).scalar_one() + assert disposal.realized_pnl_native == D(0) + assert disposal.realized_pnl_rub == D("15000.0000000000") + + row = (await _rows())[(2024, account)] + assert row.taxable_base_rub == D("15000.00") + assert row.estimated_tax_rub == D("1950.00") + # and the fact that a revaluation happened is said out loud, because a Minfin eurobond + # hiding among these would be taxed differently and cannot be detected automatically + assert any(f.check_name == "tax_currency_revaluation" for f in FINDINGS.items) + + +async def test_a_leg_without_a_rate_is_left_out_and_reported(usd_rates): + """No rate means no rouble result. Never a substitute — a finding instead.""" + account = await _broker_account() + quoted = await make_instrument(ticker="SBER", name="Сбербанк") + unquoted = await make_instrument(ticker="0700", name="Tencent", currency="HKD", board="SPBHKEX") + + await make_event( + date(2024, 2, 1), + account_id=account, + kind=EventKind.buy, + instrument_id=quoted, + quantity="10", + price="100", + amount="-1000", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=quoted, + quantity="-10", + price="150", + amount="1500", + ) + await make_event( + date(2024, 2, 1), + account_id=account, + kind=EventKind.buy, + instrument_id=unquoted, + quantity="10", + price="100", + amount="-1000", + currency="HKD", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=unquoted, + quantity="-10", + price="300", + amount="3000", + currency="HKD", + ) + await _rebuild() + + row = (await _rows())[(2024, account)] + # only the rouble trade is in the base; the HKD one did not inflate or deflate it + assert row.realized_gain_rub == D("500.00") + assert row.taxable_base_rub == D("500.00") + assert row.estimated_tax_rub == D("65.00") + assert any(f.check_name == "tax_disposal_no_fx" for f in FINDINGS.items) + + +async def test_dividends_and_coupons_are_gross_with_the_tax_that_was_withheld(app): + """`amount` is what landed and `tax` is what was taken; gross is their sum.""" + account = await _broker_account() + share = await make_instrument(ticker="SBER", name="Сбербанк") + bond = await make_instrument(ticker="SU26238", name="ОФЗ", asset_class=AssetClass.bond) + + await _payment(account, share, EventKind.dividend, date(2024, 5, 15), "870", tax="130") + await _payment(account, bond, EventKind.coupon, date(2024, 8, 1), "500") + + await _rebuild() + + row = (await _rows())[(2024, account)] + assert row.dividends_gross_rub == D("1000.00") + assert row.coupons_gross_rub == D("500.00") + assert row.tax_withheld_rub == D("130.00") + # income is outside the securities base: the agent already withheld on it + assert row.taxable_base_rub == D("0.00") + + +async def test_every_stored_number_is_a_decimal(usd_rates): + account = await _broker_account() + instrument = await make_instrument(ticker="SBER", name="Сбербанк") + await make_event( + date(2024, 2, 1), + account_id=account, + kind=EventKind.buy, + instrument_id=instrument, + quantity="10", + price="100", + amount="-1000", + ) + await make_event( + date(2024, 6, 20), + account_id=account, + kind=EventKind.sell, + instrument_id=instrument, + quantity="-10", + price="150", + amount="1500", + ) + await _rebuild() + + row = (await _rows())[(2024, account)] + for field in ( + "dividends_gross_rub", + "coupons_gross_rub", + "tax_withheld_rub", + "realized_gain_rub", + "realized_loss_rub", + "ldv_exempt_rub", + "taxable_base_rub", + "estimated_tax_rub", + "tax_rate", + ): + value = getattr(row, field) + assert isinstance(value, Decimal), field + assert not isinstance(value, float), field diff --git a/backend/tests/api/test_goals_api.py b/backend/tests/api/test_goals_api.py new file mode 100644 index 0000000..b95ef89 --- /dev/null +++ b/backend/tests/api/test_goals_api.py @@ -0,0 +1,183 @@ +"""Goal CRUD and progress over HTTP (docs/ai/phase4-contract.md §4). + +The router is mounted here rather than taken from `create_app`: wiring it into +`api/app.py` belongs to the phase-4 integration. +""" + +from collections.abc import AsyncIterator +from datetime import timedelta +from decimal import Decimal + +import pytest +from httpx import ASGITransport, AsyncClient + +from factories import make_account, make_event, make_instrument, make_price, refresh +from fintracker.analytics import today_local +from fintracker.models import AccountKind, AccountRole, AssetClass, EventKind + +D = Decimal +PREFIX = "/api/v1" + + +@pytest.fixture +async def client(app) -> AsyncIterator[AsyncClient]: + from fintracker.api.routers import goals + + app.include_router(goals.router, prefix=PREFIX) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def portfolio(app) -> None: + """A year of history ending 50 % up: 1000 shares bought at 100, now worth 150.""" + t = today_local() + start = t - timedelta(days=365) + account = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + share = await make_instrument(ticker="SBER", name="Сбербанк", asset_class=AssetClass.share) + await make_event(start, account_id=account, kind=EventKind.deposit, amount="100000") + await make_event( + start, + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="1000", + price="100", + amount="-100000", + ) + d = start + while d <= t: + await make_price(d, instrument_id=share, close="100" if d < t else "150") + d += timedelta(days=1) + await refresh() + + +async def create(client, auth_headers, **body): + payload = {"name": "Капитал", "scope": "all", "target_amount": "1000000"} | body + return await client.post(f"{PREFIX}/goals", json=payload, headers=auth_headers) + + +# --------------------------------------------------------------------------- CRUD + + +async def test_a_goal_round_trips(client, auth_headers, portfolio): + r = await create(client, auth_headers, target_date="2030-01-01", monthly_contribution="30000") + assert r.status_code == 201, r.text + body = r.json() + assert body["name"] == "Капитал" + assert Decimal(body["target_amount"]) == 1000000 + assert body["target_date"] == "2030-01-01" + assert Decimal(body["monthly_contribution"]) == 30000 + assert body["archived"] is False + + listing = await client.get(f"{PREFIX}/goals", headers=auth_headers) + assert [g["id"] for g in listing.json()] == [body["id"]] + + +async def test_a_duplicate_name_is_a_conflict(client, auth_headers, portfolio): + await create(client, auth_headers) + r = await create(client, auth_headers) + assert r.status_code == 409 + + +async def test_a_scope_the_metrics_never_built_is_refused(client, auth_headers, portfolio): + r = await create(client, auth_headers, scope="portfolio:999") + assert r.status_code == 404 + + +async def test_patch_changes_only_what_is_sent(client, auth_headers, portfolio): + created = (await create(client, auth_headers, monthly_contribution="1000")).json() + r = await client.patch( + f"{PREFIX}/goals/{created['id']}", + json={"target_amount": "500000"}, + headers=auth_headers, + ) + assert r.status_code == 200 + body = r.json() + assert Decimal(body["target_amount"]) == 500000 + assert Decimal(body["monthly_contribution"]) == 1000 + + +async def test_archived_goals_are_hidden_unless_asked_for(client, auth_headers, portfolio): + created = (await create(client, auth_headers)).json() + await client.patch( + f"{PREFIX}/goals/{created['id']}", json={"archived": True}, headers=auth_headers + ) + assert (await client.get(f"{PREFIX}/goals", headers=auth_headers)).json() == [] + shown = await client.get( + f"{PREFIX}/goals", params={"include_archived": "true"}, headers=auth_headers + ) + assert [g["id"] for g in shown.json()] == [created["id"]] + + +async def test_a_goal_can_be_deleted(client, auth_headers, portfolio): + created = (await create(client, auth_headers)).json() + r = await client.delete(f"{PREFIX}/goals/{created['id']}", headers=auth_headers) + assert r.status_code == 204 + assert (await client.get(f"{PREFIX}/goals", headers=auth_headers)).json() == [] + assert ( + await client.delete(f"{PREFIX}/goals/{created['id']}", headers=auth_headers) + ).status_code == 404 + + +async def test_goals_need_a_token(client, portfolio): + assert (await client.get(f"{PREFIX}/goals")).status_code == 401 + + +# --------------------------------------------------------------------------- progress + + +async def test_progress_is_computed_from_the_live_metrics(client, auth_headers, portfolio): + created = (await create(client, auth_headers, target_amount="300000")).json() + r = await client.get(f"{PREFIX}/goals/{created['id']}/progress", headers=auth_headers) + assert r.status_code == 200, r.text + body = r.json() + assert Decimal(body["current_value_rub"]) == 150000 + assert Decimal(body["target_amount_rub"]) == 300000 + assert Decimal(body["progress"]) == Decimal("0.5") + assert body["basis"] == "xirr" + assert body["projected_date"] is not None + assert body["projected_date"] > str(today_local()) + assert body["monthly_needed_rub"] is None + assert body["on_track"] is None + + +async def test_a_deadline_produces_a_monthly_need_and_an_on_track_flag( + client, auth_headers, portfolio +): + created = ( + await create( + client, + auth_headers, + target_amount="10000000", + target_date=str(today_local() + timedelta(days=365)), + ) + ).json() + body = ( + await client.get(f"{PREFIX}/goals/{created['id']}/progress", headers=auth_headers) + ).json() + assert Decimal(body["monthly_needed_rub"]) > 0 + assert body["on_track"] is False + + +async def test_every_money_field_is_a_string(client, auth_headers, portfolio): + created = (await create(client, auth_headers)).json() + body = ( + await client.get(f"{PREFIX}/goals/{created['id']}/progress", headers=auth_headers) + ).json() + for key in ("current_value_rub", "target_amount_rub", "progress"): + assert isinstance(body[key], str) + for key in ("assumed_rate", "monthly_needed_rub"): + assert body[key] is None or isinstance(body[key], str) + + +async def test_progress_of_an_unknown_goal_is_a_404(client, auth_headers, portfolio): + r = await client.get(f"{PREFIX}/goals/999/progress", headers=auth_headers) + assert r.status_code == 404 diff --git a/backend/tests/api/test_income_api.py b/backend/tests/api/test_income_api.py new file mode 100644 index 0000000..9340404 --- /dev/null +++ b/backend/tests/api/test_income_api.py @@ -0,0 +1,320 @@ +"""`/income` over a small portfolio: one paid dividend, one announced, one bond coupon. + +The router is not wired into `create_app` yet (that is done separately), so the fixture mounts +it on the same application the rest of the API tests use. +""" + +from collections.abc import AsyncIterator +from datetime import UTC, date, datetime, timedelta +from decimal import Decimal +from typing import Any + +import pytest +from httpx import ASGITransport, AsyncClient + +from factories import make_account, make_event, make_instrument, refresh +from fintracker.analytics import today_local +from fintracker.analytics.income import add_months, rebuild_income +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + BondNominalSchedule, + CorporateAction, + CorporateActionKind, + CorporateActionStatus, + Event, + EventKind, + Instrument, +) + +D = Decimal +PREFIX = "/api/v1/income" + + +async def make_bond(*, ticker: str, maturity: date | None = None) -> int: + async with get_sessionmaker()() as session: + bond = Instrument( + asset_class=AssetClass.bond, + ticker=ticker, + board="TQOB", + name=ticker, + currency="RUB", + nominal=D(1000), + nominal_currency="RUB", + maturity_date=maturity, + ) + session.add(bond) + await session.commit() + await session.refresh(bond) + return bond.id + + +async def make_payout( + d: date, + *, + account_id: int, + instrument_id: int, + amount: str, + kind: EventKind = EventKind.dividend, + tax: str | None = None, +) -> None: + async with get_sessionmaker()() as session: + session.add( + Event( + account_id=account_id, + instrument_id=instrument_id, + kind=kind, + ts=datetime.combine(d, datetime.min.time(), tzinfo=UTC), + trade_date=d, + amount=D(amount), + currency="RUB", + tax=D(tax) if tax is not None else None, + tax_currency="RUB" if tax is not None else None, + source="tinvest", + source_id=f"pay-{instrument_id}-{d}-{amount}", + dedupe_key=f"tinvest:pay-{instrument_id}-{d}-{amount}", + ) + ) + await session.commit() + + +async def make_action( + *, + instrument_id: int, + kind: CorporateActionKind, + status: CorporateActionStatus, + pay_date: date, + record_date: date | None = None, + amount_per_unit: str, +) -> None: + async with get_sessionmaker()() as session: + session.add( + CorporateAction( + instrument_id=instrument_id, + kind=kind, + status=status, + pay_date=pay_date, + record_date=record_date, + amount_per_unit=D(amount_per_unit), + currency="RUB", + source="moex", + source_id=f"{kind}-{pay_date}", + ) + ) + await session.commit() + + +async def make_nominal(instrument_id: int, effective: date, nominal: str) -> None: + async with get_sessionmaker()() as session: + session.add( + BondNominalSchedule( + instrument_id=instrument_id, + effective_date=effective, + nominal=D(nominal), + currency="RUB", + source="moex", + ) + ) + await session.commit() + + +async def rebuild() -> None: + async with get_sessionmaker()() as session: + await rebuild_income(session) + await session.commit() + + +@pytest.fixture +async def income_client(app, user) -> AsyncIterator[AsyncClient]: + from fintracker.api.routers import income + + app.include_router(income.router, prefix="/api/v1") + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + r = await c.post("/api/v1/auth/login", json=user) + assert r.status_code == 200, r.text + c.headers["Authorization"] = f"Bearer {r.json()['access_token']}" + yield c + + +@pytest.fixture +async def portfolio(app) -> dict[str, int]: + """A share that paid twice and has a declared payout, and an amortising bond.""" + today = today_local() + account = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + share = await make_instrument(ticker="SBER", name="Сбербанк России") + bond = await make_bond(ticker="RU000API", maturity=add_months(today, 30)) + + await make_event( + add_months(today, -20), + account_id=account, + kind=EventKind.buy, + instrument_id=share, + quantity="20", + price="250", + amount="-5000", + ) + await make_event( + add_months(today, -20), + account_id=account, + kind=EventKind.buy, + instrument_id=bond, + quantity="10", + price="1000", + amount="-10000", + ) + await make_payout( + add_months(today, -13), account_id=account, instrument_id=share, amount="696.80", tax="104" + ) + await make_payout( + add_months(today, -1), account_id=account, instrument_id=share, amount="696.80", tax="104" + ) + await make_action( + instrument_id=share, + kind=CorporateActionKind.dividend, + status=CorporateActionStatus.announced, + record_date=today + timedelta(days=21), + pay_date=today + timedelta(days=24), + amount_per_unit="34.84", + ) + await make_nominal(bond, add_months(today, -24), "1000") + await make_action( + instrument_id=bond, + kind=CorporateActionKind.coupon, + status=CorporateActionStatus.announced, + pay_date=add_months(today, 2), + amount_per_unit="40", + ) + await refresh() + await rebuild() + return {"account": account, "share": share, "bond": bond} + + +def floats(value: Any, path: str = "$") -> list[str]: + """Every place a float leaked into the payload — money must travel as a string.""" + if isinstance(value, bool): + return [] + if isinstance(value, float): + return [path] + if isinstance(value, dict): + return [p for k, v in value.items() for p in floats(v, f"{path}.{k}")] + if isinstance(value, list): + return [p for i, v in enumerate(value) for p in floats(v, f"{path}[{i}]")] + return [] + + +async def test_calendar_shows_the_future_with_money_as_strings_and_a_basis_on_every_row( + income_client: AsyncClient, portfolio: dict[str, int] +): + r = await income_client.get(f"{PREFIX}/calendar") + assert r.status_code == 200, r.text + body = r.json() + assert floats(body) == [] + + assert body["currency"] == "RUB" + assert body["entries"], body + for entry in body["entries"]: + assert isinstance(entry["amount"], str) + assert isinstance(entry["qty"], str) + assert entry["basis"] in {"schedule", "announced", "history"} + + by_key = {(e["kind"], e["basis"]): e for e in body["entries"]} + # the declared autumn dividend, and next year's payment that only history knows about + announced = by_key[("dividend", "announced")] + assert announced["amount"] == "696.8000000000" + assert announced["ticker"] == "SBER" + assert announced["per_unit"] == "34.8400000000" + assert ("dividend", "history") in by_key + assert by_key[("coupon", "schedule")]["amount"] == "400.0000000000" + + assert set(body["by_basis"]) == {"announced", "history", "schedule"} + total = sum(D(v) for v in body["by_basis"].values()) + assert D(body["total_expected_rub"]) == total + + +async def test_include_paid_adds_the_history_rows_and_nothing_else( + income_client: AsyncClient, portfolio: dict[str, int] +): + today = today_local() + window = {"date_from": str(add_months(today, -24)), "date_to": str(add_months(today, 12))} + + without = (await income_client.get(f"{PREFIX}/calendar", params=window)).json() + assert {e["basis"] for e in without["entries"]} == {"announced", "history", "schedule"} + + with_paid = ( + await income_client.get(f"{PREFIX}/calendar", params={**window, "include_paid": "true"}) + ).json() + paid = [e for e in with_paid["entries"] if e["basis"] == "paid"] + assert len(paid) == 2 + assert paid[0]["tax_withheld"] == "104.0000000000" + # a payment already received is not an expectation: the totals must not move + assert with_paid["total_expected_rub"] == without["total_expected_rub"] + assert "paid" not in with_paid["by_basis"] + + +async def test_history_groups_by_month_and_totals_the_tax( + income_client: AsyncClient, portfolio: dict[str, int] +): + r = await income_client.get(f"{PREFIX}/history") + assert r.status_code == 200, r.text + body = r.json() + assert floats(body) == [] + assert [row["kind"] for row in body["rows"]] == ["dividend", "dividend"] + assert {row["payment_count"] for row in body["rows"]} == {1} + assert D(body["totals"]["amount_rub"]) == D("1393.60") + assert D(body["totals"]["tax_withheld_rub"]) == D("208") + + filtered = (await income_client.get(f"{PREFIX}/history", params={"kind": "coupon"})).json() + assert filtered["rows"] == [] + + +async def test_forecast_splits_every_month_by_basis( + income_client: AsyncClient, portfolio: dict[str, int] +): + r = await income_client.get(f"{PREFIX}/forecast", params={"months": 12}) + assert r.status_code == 200, r.text + body = r.json() + assert floats(body) == [] + assert body["months"], body + + bases = {basis for month in body["months"] for basis in month["by_basis"]} + assert bases <= {"schedule", "announced", "history"} + for month in body["months"]: + assert D(month["amount_rub"]) == sum(D(v) for v in month["by_basis"].values()) + assert D(body["total_rub"]) == sum(D(m["amount_rub"]) for m in body["months"]) + assert isinstance(body["warnings"], list) + + +async def test_forecast_rejects_a_horizon_the_table_was_not_built_for( + income_client: AsyncClient, portfolio: dict[str, int] +): + assert (await income_client.get(f"{PREFIX}/forecast", params={"months": 0})).status_code == 422 + assert (await income_client.get(f"{PREFIX}/forecast", params={"months": 37})).status_code == 422 + + +async def test_an_unknown_scope_is_a_404_not_an_empty_calendar( + income_client: AsyncClient, portfolio: dict[str, int] +): + r = await income_client.get(f"{PREFIX}/calendar", params={"scope": "account:999"}) + assert r.status_code == 404 + + +async def test_the_endpoints_require_a_token(income_client: AsyncClient, app): + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as anon: + assert (await anon.get(f"{PREFIX}/calendar")).status_code == 401 + + +async def test_the_endpoints_are_pure_reads_and_repeat_themselves( + income_client: AsyncClient, portfolio: dict[str, int] +): + first = (await income_client.get(f"{PREFIX}/forecast")).json() + second = (await income_client.get(f"{PREFIX}/forecast")).json() + assert first == second diff --git a/backend/tests/api/test_rebalance_api.py b/backend/tests/api/test_rebalance_api.py new file mode 100644 index 0000000..9a85685 --- /dev/null +++ b/backend/tests/api/test_rebalance_api.py @@ -0,0 +1,285 @@ +"""Target weights and rebalancing over HTTP (docs/ai/phase4-contract.md §2). + +The router is mounted here rather than taken from `create_app`: wiring it into +`api/app.py` belongs to the phase-4 integration, and these tests should not wait on it. +""" + +from collections.abc import AsyncIterator +from datetime import timedelta +from decimal import Decimal + +import pytest +from httpx import ASGITransport, AsyncClient + +from factories import make_account, make_event, make_instrument, make_price, refresh +from fintracker.analytics import today_local +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + EventKind, + Instrument, + Portfolio, + PortfolioAccount, +) + +D = Decimal +PREFIX = "/api/v1" + + +@pytest.fixture +async def client(app) -> AsyncIterator[AsyncClient]: + from fintracker.api.routers import rebalance + + app.include_router(rebalance.router, prefix=PREFIX) + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + yield c + + +@pytest.fixture +async def portfolio(app) -> dict[str, int]: + """500 SBER (lot 10) at 100, 20 ОФЗ at 1000, 30 000 ₽ left in cash.""" + t = today_local() + bought = t - timedelta(days=40) + account = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + sber = await make_instrument(ticker="SBER", name="Сбербанк", asset_class=AssetClass.share) + ofz = await make_instrument(ticker="OFZ", name="ОФЗ", asset_class=AssetClass.bond) + + async with get_sessionmaker()() as session: + instrument = await session.get(Instrument, sber) + assert instrument is not None + instrument.lot = 10 + portfolio = Portfolio(name="Основной") + session.add(portfolio) + await session.flush() + session.add(PortfolioAccount(portfolio_id=portfolio.id, account_id=account)) + portfolio_id = portfolio.id + await session.commit() + + await make_event(bought, account_id=account, kind=EventKind.deposit, amount="100000") + await make_event( + bought, + account_id=account, + kind=EventKind.buy, + instrument_id=sber, + quantity="500", + price="100", + amount="-50000", + ) + await make_event( + bought, + account_id=account, + kind=EventKind.buy, + instrument_id=ofz, + quantity="20", + price="1000", + amount="-20000", + ) + d = bought + while d <= t: + await make_price(d, instrument_id=sber, close="100") + await make_price(d, instrument_id=ofz, close="1000") + d += timedelta(days=1) + await refresh() + return {"portfolio": portfolio_id, "sber": sber, "ofz": ofz} + + +def targets(*rows: tuple[str, str, str]) -> dict: + return { + "dimension": "asset_class", + "targets": [{"bucket": b, "target_weight": w, "band": band} for b, w, band in rows], + } + + +async def put(client, auth_headers, portfolio_id: int, body: dict): + return await client.put( + f"{PREFIX}/portfolios/{portfolio_id}/targets", json=body, headers=auth_headers + ) + + +# --------------------------------------------------------------------------- targets + + +async def test_targets_round_trip_and_report_their_sum(client, auth_headers, portfolio): + r = await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")), + ) + assert r.status_code == 200, r.text + body = r.json() + assert Decimal(body["weights_sum"]) == 1 + assert [t["bucket"] for t in body["targets"]] == ["bond", "cash", "share"] + assert Decimal(body["targets"][0]["target_weight"]) == Decimal("0.2") + + r = await client.get( + f"{PREFIX}/portfolios/{portfolio['portfolio']}/targets", headers=auth_headers + ) + assert r.status_code == 200 + assert r.json() == body + + +async def test_weights_that_do_not_add_up_are_refused_with_the_actual_sum( + client, auth_headers, portfolio +): + r = await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.01"), ("bond", "0.3", "0.01")), + ) + assert r.status_code == 422 + body = r.json() + assert "0.9" in body["detail"] + assert Decimal(body["weights_sum"]) == Decimal("0.9") + + +async def test_a_set_is_replaced_whole_not_merged(client, auth_headers, portfolio): + await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")), + ) + r = await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.7", "0.01"), ("cash", "0.3", "0.01")), + ) + assert r.status_code == 200 + assert [t["bucket"] for t in r.json()["targets"]] == ["cash", "share"] + + +async def test_a_duplicated_bucket_is_refused(client, auth_headers, portfolio): + r = await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.5", "0.01"), ("share", "0.5", "0.01")), + ) + assert r.status_code == 422 + assert "share" in r.json()["detail"] + + +async def test_an_unknown_dimension_is_refused(client, auth_headers, portfolio): + body = targets(("share", "1", "0.01")) + body["dimension"] = "mood" + r = await put(client, auth_headers, portfolio["portfolio"], body) + assert r.status_code == 422 + + +async def test_an_unknown_portfolio_is_a_404(client, auth_headers, portfolio): + r = await client.get(f"{PREFIX}/portfolios/999/targets", headers=auth_headers) + assert r.status_code == 404 + + +async def test_targets_need_a_token(client, portfolio): + r = await client.get(f"{PREFIX}/portfolios/{portfolio['portfolio']}/targets") + assert r.status_code == 401 + + +# --------------------------------------------------------------------------- suggestions + + +async def test_the_suggestion_respects_the_lot_and_the_cash(client, auth_headers, portfolio): + await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")), + ) + r = await client.get( + f"{PREFIX}/portfolios/{portfolio['portfolio']}/rebalance", headers=auth_headers + ) + assert r.status_code == 200, r.text + body = r.json() + assert Decimal(body["total_value_rub"]) == 100000 + assert Decimal(body["cash_available_rub"]) == 30000 + + share = next(b for b in body["buckets"] if b["bucket"] == "share") + assert Decimal(share["current_weight"]) == Decimal("0.5") + assert Decimal(share["target_weight"]) == Decimal("0.6") + assert Decimal(share["drift"]) == Decimal("-0.1") + assert share["within_band"] is False + trade = share["trades"][0] + assert trade["action"] == "buy" + assert trade["lot"] == 10 + assert Decimal(trade["suggested_qty"]) % 10 == 0 + assert Decimal(trade["suggested_qty"]) == 100 + assert trade["blocked_by_cash"] is False + + bond = next(b for b in body["buckets"] if b["bucket"] == "bond") + assert bond["within_band"] is True + assert bond["trades"] == [] + + +async def test_the_what_if_cash_blocks_the_buy(client, auth_headers, portfolio): + await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")), + ) + r = await client.get( + f"{PREFIX}/portfolios/{portfolio['portfolio']}/rebalance", + params={"cash_available": "2500"}, + headers=auth_headers, + ) + assert r.status_code == 200 + trade = next(t for b in r.json()["buckets"] for t in b["trades"] if b["bucket"] == "share") + assert Decimal(trade["suggested_qty"]) == 20 + assert trade["blocked_by_cash"] is True + + +async def test_a_wide_band_silences_every_suggestion(client, auth_headers, portfolio): + await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.5"), ("bond", "0.2", "0.5"), ("cash", "0.2", "0.5")), + ) + r = await client.get( + f"{PREFIX}/portfolios/{portfolio['portfolio']}/rebalance", headers=auth_headers + ) + body = r.json() + assert all(b["within_band"] for b in body["buckets"] if b["target_weight"] is not None) + assert all(t["suggested_qty"] is None for b in body["buckets"] for t in b["trades"]) + + +async def test_without_targets_there_is_nothing_to_rebalance(client, auth_headers, portfolio): + r = await client.get( + f"{PREFIX}/portfolios/{portfolio['portfolio']}/rebalance", headers=auth_headers + ) + assert r.status_code == 200 + assert all(b["target_weight"] is None for b in r.json()["buckets"]) + + +async def test_every_money_field_is_a_string(client, auth_headers, portfolio): + await put( + client, + auth_headers, + portfolio["portfolio"], + targets(("share", "0.6", "0.01"), ("bond", "0.2", "0.01"), ("cash", "0.2", "0.01")), + ) + r = await client.get( + f"{PREFIX}/portfolios/{portfolio['portfolio']}/rebalance", headers=auth_headers + ) + body = r.json() + for key in ("total_value_rub", "cash_available_rub"): + assert isinstance(body[key], str) + for b in body["buckets"]: + for key in ("current_value_rub", "current_weight", "delta_value_rub"): + assert isinstance(b[key], str) + for t in b["trades"]: + for key in ("suggested_qty", "price", "amount_rub"): + assert t[key] is None or isinstance(t[key], str) diff --git a/backend/tests/api/test_tax_api.py b/backend/tests/api/test_tax_api.py new file mode 100644 index 0000000..9fb1e77 --- /dev/null +++ b/backend/tests/api/test_tax_api.py @@ -0,0 +1,144 @@ +"""`GET /tax` and `GET /tax/lots` — the screen that prices selling before the three-year mark. + +The router is not wired into `api/app.py` by this module's author, so the tests mount it on a +copy of the application. That keeps the check honest about the routes' own behaviour while +leaving the inclusion order to whoever owns `app.py`. +""" + +import json +from collections.abc import AsyncIterator +from datetime import timedelta +from decimal import Decimal + +import pytest +from httpx import ASGITransport, AsyncClient + +from factories import make_account, make_event, make_instrument, make_price +from fintracker.analytics import today_local +from fintracker.analytics.tax import rebuild_tax_year +from fintracker.api.routers import tax as tax_router +from fintracker.db import get_sessionmaker +from fintracker.ledger.rebuild import rebuild_lots +from fintracker.models import AccountKind, AccountRole, AssetClass, EventKind +from fintracker.pricing.fx import rebuild_fx_daily + +D = Decimal +LDV_DAYS = 3 * 365 + + +@pytest.fixture +async def tax_client(app, auth_headers) -> AsyncIterator[AsyncClient]: + """The application plus the tax router, which `app.py` does not include yet.""" + app.include_router(tax_router.router, prefix="/api/v1") + async with AsyncClient(transport=ASGITransport(app=app), base_url="http://test") as c: + c.headers.update(auth_headers) + yield c + + +@pytest.fixture +async def lots(app) -> dict[str, object]: + """Two open lots of the same paper: one past the ЛДВ mark, one still short of it.""" + t = today_local() + old_date = t - timedelta(days=LDV_DAYS + 30) + young_date = t - timedelta(days=400) + + account = await make_account( + name="ИИС", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + ) + old_share = await make_instrument(ticker="SBER", name="Сбербанк", asset_class=AssetClass.share) + young_share = await make_instrument(ticker="GAZP", name="Газпром", asset_class=AssetClass.share) + await make_event( + old_date, + account_id=account, + kind=EventKind.buy, + instrument_id=old_share, + quantity="10", + price="100", + amount="-1000", + ) + await make_event( + young_date, + account_id=account, + kind=EventKind.buy, + instrument_id=young_share, + quantity="20", + price="240", + amount="-4800", + ) + await make_price(t, instrument_id=old_share, close="150") + await make_price(t, instrument_id=young_share, close="275.89") + + async with get_sessionmaker()() as session: + await rebuild_fx_daily(session) + await session.commit() + await rebuild_lots(session) + await session.commit() + await rebuild_tax_year(session) + await session.commit() + return { + "account": account, + "old": old_share, + "young": young_share, + "old_date": old_date, + "young_date": young_date, + "today": t, + } + + +async def test_lots_show_the_days_left_to_the_exemption(tax_client, lots): + r = await tax_client.get("/api/v1/tax/lots") + assert r.status_code == 200, r.text + body = r.json() + assert body["estimated"] is True + assert body["tax_rate"] == "0.13" + assert body["disclaimer"] + + by_ticker = {lot["ticker"]: lot for lot in body["lots"]} + young = by_ticker["GAZP"] + ldv_date = lots["young_date"] + timedelta(days=LDV_DAYS) + assert young["ldv_date"] == ldv_date.isoformat() + assert young["days_to_ldv"] == (ldv_date - lots["today"]).days + assert young["ldv_eligible"] is False + # 20 x 275.89 = 5517.80 against a cost of 4800 -> 717.80 unrealised, 13 % of it is 93.31 + assert young["market_value_rub"] == "5517.80" + assert young["cost_rub"] == "4800.00" + assert young["unrealized_gain_rub"] == "717.80" + assert young["tax_if_sold_now_rub"] == "93.31" + + +async def test_a_lot_past_three_years_costs_nothing_to_sell(tax_client, lots): + body = (await tax_client.get("/api/v1/tax/lots")).json() + old = {lot["ticker"]: lot for lot in body["lots"]}["SBER"] + + assert old["ldv_eligible"] is True + assert old["days_to_ldv"] == 0 + # 10 x 150 = 1500 against 1000 is a real gain, and art. 219.1 makes it untaxed + assert old["unrealized_gain_rub"] == "500.00" + assert old["tax_if_sold_now_rub"] == "0.00" + + +async def test_the_year_summary_is_marked_an_estimate(tax_client, lots): + r = await tax_client.get("/api/v1/tax", params={"year": lots["today"].year}) + assert r.status_code == 200, r.text + body = r.json() + assert body["estimated"] is True + assert body["tax_rate"] == "0.13" + assert "брокер" in body["disclaimer"] + # nothing was sold, so the year is empty — but it is still a well-formed answer + assert body["totals"]["estimated_tax_rub"] == "0.00" + assert body["accounts"] == [] + + +async def test_no_float_reaches_the_wire(tax_client, lots): + def reject_float(raw: str) -> None: + raise AssertionError(f"float in the response: {raw}") + + for path in ("/api/v1/tax", "/api/v1/tax/lots"): + r = await tax_client.get(path) + assert r.status_code == 200, r.text + json.loads(r.text, parse_float=reject_float) diff --git a/backend/tests/pricing/__init__.py b/backend/tests/pricing/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/tests/pricing/test_payout_merge.py b/backend/tests/pricing/test_payout_merge.py new file mode 100644 index 0000000..61a74a6 --- /dev/null +++ b/backend/tests/pricing/test_payout_merge.py @@ -0,0 +1,204 @@ +"""Which feed wins when both describe the same payout, and what happens to the difference. + +These run on plain `CorporateAction` objects with no session: the priority rule is a read +rule, and `analytics/income.py` has to be able to apply it to whatever it already loaded. +""" + +from __future__ import annotations + +from datetime import date +from decimal import Decimal + +from fintracker.analytics import FINDINGS +from fintracker.models.pricing import CorporateAction, CorporateActionKind, CorporateActionStatus +from fintracker.pricing.payouts import ( + nominal_outranks, + resolve, + resolve_payouts, + weaker_or_equal_nominal_sources, +) + +DIV = CorporateActionKind.dividend +CPN = CorporateActionKind.coupon + + +def action( + *, + kind=DIV, + source="tinvest", + amount: str | None = "10", + status=CorporateActionStatus.announced, + instrument_id=1, + record_date=None, + ex_date=None, + pay_date=None, + currency="RUB", + source_id=None, +) -> CorporateAction: + return CorporateAction( + instrument_id=instrument_id, + kind=kind, + status=status, + record_date=record_date, + ex_date=ex_date, + pay_date=pay_date, + amount_per_unit=None if amount is None else Decimal(amount), + currency=currency, + ratio=None, + source=source, + source_id=source_id or f"{source}-1", + ) + + +def test_bond_coupon_is_read_from_moex(): + """bondization is the issuer's registered schedule; T-Invest answers for a window.""" + moex = action(kind=CPN, source="moex", amount="34.90", pay_date=date(2026, 11, 5)) + tinvest = action(kind=CPN, source="tinvest", amount="34.90", pay_date=date(2026, 11, 5)) + + resolved = resolve_payouts([tinvest, moex], report=False) + + assert len(resolved) == 1 + assert resolved[0].source == "moex" + + +def test_share_dividend_is_read_from_tinvest(): + """T-Invest states what will settle on the account; MOEX states the register.""" + moex = action(source="moex", amount="52", record_date=date(2026, 7, 10)) + tinvest = action( + source="tinvest", amount="52", record_date=date(2026, 7, 10), pay_date=date(2026, 7, 24) + ) + + resolved = resolve_payouts([moex, tinvest], report=False) + + assert len(resolved) == 1 + assert resolved[0].source == "tinvest" + + +def test_feeds_merge_on_the_record_date_they_share(): + """MOEX states only the register date and T-Invest also states the payment date. + + Keying the merge on the payment date alone would leave them in separate buckets and + count the dividend twice. + """ + moex = action(source="moex", amount="52", record_date=date(2026, 7, 10)) + tinvest = action( + source="tinvest", amount="52", record_date=date(2026, 7, 10), pay_date=date(2026, 7, 24) + ) + + assert len(resolve_payouts([moex, tinvest], report=False)) == 1 + + +def test_two_different_payouts_of_one_paper_stay_two(): + interim = action(source="tinvest", amount="17", record_date=date(2026, 1, 12)) + final = action(source="tinvest", amount="52", record_date=date(2026, 7, 10), source_id="t-2") + + assert len(resolve_payouts([interim, final], report=False)) == 2 + + +def test_a_mismatch_in_amount_produces_a_finding_and_keeps_the_winner(): + """The winner is still the winner; the gap goes to the quality report, not to /dev/null.""" + FINDINGS.reset() + tinvest = action(source="tinvest", amount="45.24", record_date=date(2026, 7, 10)) + moex = action(source="moex", amount="52.00", record_date=date(2026, 7, 10)) + + resolved = resolve_payouts([moex, tinvest]) + + assert [a.source for a in resolved] == ["tinvest"] + assert resolved[0].amount_per_unit == Decimal("45.24") + finding = next(f for f in FINDINGS.items if f.check_name == "payout_amount_mismatch") + assert finding.severity == "warn" + assert finding.ref == {"instruments": [1]} + FINDINGS.reset() + + +def test_amounts_within_rounding_are_not_a_mismatch(): + FINDINGS.reset() + tinvest = action(source="tinvest", amount="34.90", pay_date=date(2026, 11, 5), kind=CPN) + moex = action(source="moex", amount="34.9000000001", pay_date=date(2026, 11, 5), kind=CPN) + + resolve_payouts([tinvest, moex]) + + assert FINDINGS.items == [] + + +def test_a_missing_amount_is_a_gap_not_a_disagreement(): + """A floating coupon whose rate is unfixed arrives dated and priceless — every refresh.""" + FINDINGS.reset() + moex = action(source="moex", amount=None, pay_date=date(2027, 2, 5), kind=CPN) + tinvest = action(source="tinvest", amount="34.90", pay_date=date(2027, 2, 5), kind=CPN) + + resolved = resolve_payouts([moex, tinvest]) + + assert resolved[0].source == "moex" + assert FINDINGS.items == [] + + +def test_a_paid_row_outranks_an_announcement_from_the_stronger_feed(): + """Money that has moved beats a feed's announcement of the same payout, either way round.""" + ledger = action( + kind=CPN, + source="tinvest", + amount="34.90", + status=CorporateActionStatus.paid, + pay_date=date(2026, 5, 5), + ) + announced = action( + kind=CPN, + source="moex", + amount="34.90", + status=CorporateActionStatus.announced, + pay_date=date(2026, 5, 5), + ) + + resolved = resolve_payouts([announced, ledger], report=False) + + assert [(a.source, a.status) for a in resolved] == [("tinvest", CorporateActionStatus.paid)] + + +def test_different_papers_never_merge(): + a = action(source="moex", record_date=date(2026, 7, 10), instrument_id=1) + b = action(source="moex", record_date=date(2026, 7, 10), instrument_id=2) + + assert len(resolve_payouts([a, b], report=False)) == 2 + + +def test_an_undated_row_survives_on_its_own(): + dated = action(source="tinvest", record_date=date(2026, 7, 10)) + undated = action(source="tinvest", amount="9", source_id="t-2") + + assert len(resolve_payouts([dated, undated], report=False)) == 2 + + +def test_conflicts_are_reported_per_group_not_summed_into_one_line(): + result = resolve( + [ + action(source="tinvest", amount="45", record_date=date(2026, 7, 10)), + action(source="moex", amount="52", record_date=date(2026, 7, 10)), + action(source="tinvest", amount="10", record_date=date(2026, 1, 9), source_id="t-2"), + action(source="moex", amount="12", record_date=date(2026, 1, 9), source_id="m-2"), + ] + ) + + assert len(result.payouts) == 2 + assert len(result.conflicts) == 2 + + +def test_nominal_schedule_precedence_is_a_write_rule(): + """`bond_nominal_schedule` is keyed without `source`, so the two feeds share one row.""" + assert nominal_outranks("moex", "tinvest") + assert not nominal_outranks("tinvest", "moex") + assert nominal_outranks("moex", "moex") # a feed must be able to correct itself + assert set(weaker_or_equal_nominal_sources("moex")) == {"moex", "tinvest", "ledger"} + assert set(weaker_or_equal_nominal_sources("tinvest")) == {"tinvest", "ledger"} + + +def test_no_float_anywhere_in_a_resolution(): + resolved = resolve_payouts( + [ + action(source="tinvest", amount="45.24", record_date=date(2026, 7, 10)), + action(source="moex", amount="52.00", record_date=date(2026, 7, 10)), + ], + report=False, + ) + assert all(not isinstance(a.amount_per_unit, float) for a in resolved) + assert isinstance(resolved[0].amount_per_unit, Decimal) diff --git a/backend/tests/sources/test_moex_payouts.py b/backend/tests/sources/test_moex_payouts.py new file mode 100644 index 0000000..86f2c98 --- /dev/null +++ b/backend/tests/sources/test_moex_payouts.py @@ -0,0 +1,365 @@ +"""MOEX ISS as the second payout feed: bondization and the dividend register. + +Every request goes through respx — the ISS is never actually called. +""" + +from __future__ import annotations + +from datetime import date, timedelta +from decimal import Decimal + +import httpx +import respx +from sqlalchemy import select + +from factories import make_account, make_event +from fintracker.analytics import today_local +from fintracker.config import Settings +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + BondNominalSchedule, + CorporateAction, + CorporateActionKind, + CorporateActionStatus, + EventKind, + Instrument, +) +from fintracker.sources.moex.client import AmortisationRow, CouponRow, MoexClient +from fintracker.sources.moex.payouts import ( + MoexDividendRow, + MoexPayoutsSource, + coupon_payout, + dividend_payout, + fetch_dividends, + nominal_schedule, +) + +ISS = "https://iss.moex.com/iss" +D = Decimal +TODAY = today_local() +PAST = TODAY - timedelta(days=30) +FUTURE = TODAY + timedelta(days=30) + + +def block(name: str, columns: list[str], data: list[list]) -> dict: + return {name: {"columns": columns, "data": data}} + + +def bondization(coupons: list[list], amortisations: list[list] | None = None) -> dict: + return { + "coupons": { + "columns": ["coupondate", "value", "valueprc", "faceunit"], + "data": coupons, + }, + "amortizations": { + "columns": ["amortdate", "value", "facevalue", "faceunit"], + "data": amortisations or [], + }, + } + + +# --- mapping ------------------------------------------------------------------------- + + +def test_a_coupon_row_maps_to_a_moex_sourced_corporate_action(): + payout = coupon_payout( + CouponRow(coupon_date=FUTURE, value=D("34.90"), value_pct=D("7.0"), currency="RUB"), + instrument_id=9, + today=TODAY, + ) + + assert payout is not None + assert payout.kind is CorporateActionKind.coupon + assert payout.source == "moex" + assert payout.source_id == f"cpn:{FUTURE.isoformat()}" + assert payout.pay_date == FUTURE + assert payout.amount_per_unit == D("34.90") + assert payout.currency == "RUB" + assert payout.status is CorporateActionStatus.announced + + +def test_a_coupon_already_paid_is_stored_as_paid(): + payout = coupon_payout( + CouponRow(coupon_date=PAST, value=D("34.90"), value_pct=None, currency="RUB"), + instrument_id=9, + today=TODAY, + ) + assert payout is not None + assert payout.status is CorporateActionStatus.paid + + +def test_a_floating_coupon_without_a_rate_keeps_its_date_and_loses_its_amount(): + """The date is published long before the rate is fixed, and the calendar needs it.""" + payout = coupon_payout( + CouponRow(coupon_date=FUTURE, value=None, value_pct=None, currency="RUB"), + instrument_id=9, + today=TODAY, + ) + assert payout is not None + assert payout.amount_per_unit is None + + +def test_a_dividend_register_row_states_the_record_date_and_nothing_else(): + payout = dividend_payout( + MoexDividendRow( + secid="SBER", registry_close_date=date(2026, 7, 10), value=D("52"), currency="RUB" + ), + instrument_id=7, + today=TODAY, + ) + + assert payout is not None + assert payout.record_date == date(2026, 7, 10) + assert payout.pay_date is None + assert payout.source_id == "div:2026-07-10" + assert payout.amount_per_unit == D("52") + + +def test_amortisations_run_the_nominal_down_to_zero(): + warnings: list[str] = [] + points = nominal_schedule( + [ + AmortisationRow( + amort_date=date(2028, 5, 5), value=D(500), face_value=D(500), currency="SUR" + ), + AmortisationRow( + amort_date=date(2026, 5, 5), value=D(250), face_value=D(1000), currency="SUR" + ), + AmortisationRow( + amort_date=date(2027, 5, 5), value=D(250), face_value=D(750), currency="SUR" + ), + ], + instrument_id=9, + currency="RUB", + warnings=warnings, + ) + + assert [(p.effective_date, p.nominal) for p in points] == [ + (date(2026, 5, 5), D(750)), + (date(2027, 5, 5), D(500)), + (date(2028, 5, 5), D(0)), + ] + assert all(p.source == "moex" for p in points) + assert warnings == [] + + +def test_a_plan_that_does_not_add_up_to_the_stated_nominal_is_reported(): + warnings: list[str] = [] + nominal_schedule( + [ + AmortisationRow( + amort_date=date(2026, 5, 5), value=D(250), face_value=D(1000), currency="SUR" + ), + AmortisationRow( + amort_date=date(2027, 5, 5), value=D(250), face_value=D(750), currency="SUR" + ), + ], + instrument_id=9, + currency="RUB", + warnings=warnings, + secid="RU000A", + ) + + assert len(warnings) == 1 + assert "1000" in warnings[0] + + +def test_nominal_points_are_decimal_never_float(): + points = nominal_schedule( + [ + AmortisationRow( + amort_date=date(2026, 5, 5), value=D("1000"), face_value=None, currency=None + ) + ], + instrument_id=9, + currency="RUB", + warnings=[], + ) + assert all(isinstance(p.nominal, Decimal) and not isinstance(p.nominal, float) for p in points) + + +# --- client -------------------------------------------------------------------------- + + +@respx.mock +async def test_bondization_gives_the_whole_coupon_schedule_not_just_the_near_ones(): + """This is why bonds are read from MOEX: the plan runs to maturity, in one request.""" + schedule = [ + [(TODAY + timedelta(days=30 * n)).isoformat(), 34.9, 7.0, "SUR"] for n in range(1, 25) + ] + respx.get(url__startswith=f"{ISS}/securities/RU000A/bondization").mock( + return_value=httpx.Response(200, json=bondization(schedule)) + ) + + async with MoexClient() as moex: + coupons, amortisations = await moex.bondization("RU000A") + + within_a_year = [ + c for c in coupons if c.coupon_date and c.coupon_date <= TODAY + timedelta(days=365) + ] + assert len(within_a_year) >= 12 + assert all(c.value == D("34.9") for c in within_a_year) + assert amortisations == [] + + +@respx.mock +async def test_the_dividend_extract_is_read_by_column_name(): + respx.get(url__startswith=f"{ISS}/securities/SBER/dividends").mock( + return_value=httpx.Response( + 200, + json=block( + "dividends", + ["secid", "isin", "registryclosedate", "value", "currencyid"], + [["SBER", "RU0009029540", "2026-07-10", 34.84, "SUR"]], + ), + ) + ) + + async with httpx.AsyncClient(trust_env=False) as http: + rows = await fetch_dividends(http, "SBER") + + assert rows == [ + MoexDividendRow( + secid="SBER", + registry_close_date=date(2026, 7, 10), + value=D("34.84"), + currency="RUB", # ISS says SUR + ) + ] + + +# --- sync ---------------------------------------------------------------------------- + + +async def seed(asset_class: AssetClass, ticker: str) -> int: + account_id = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + source_id=f"tinv-{ticker}", + ) + async with get_sessionmaker()() as session: + instrument = Instrument( + asset_class=asset_class, + ticker=ticker, + board="TQBR", + name=ticker, + currency="RUB", + ) + session.add(instrument) + await session.commit() + await session.refresh(instrument) + instrument_id = instrument.id + await make_event( + PAST, account_id=account_id, kind=EventKind.buy, instrument_id=instrument_id, quantity=10 + ) + return instrument_id + + +async def rows_of(model, **where): + async with get_sessionmaker()() as session: + stmt = select(model) + for column, value in where.items(): + stmt = stmt.where(getattr(model, column) == value) + return list((await session.execute(stmt)).scalars().all()) + + +def mock_iss(mock_http) -> None: + mock_http.get(url__startswith=f"{ISS}/securities/RU000A/bondization").mock( + return_value=httpx.Response( + 200, + json=bondization( + [ + ["2027-05-05", 34.9, 7.0, "SUR"], + ["2028-05-05", 34.9, 7.0, "SUR"], + ], + [ + ["2027-05-05", 250, 1000, "SUR"], + ["2028-05-05", 750, 750, "SUR"], + ], + ), + ) + ) + mock_http.get(url__startswith=f"{ISS}/securities/SBER/dividends").mock( + return_value=httpx.Response( + 200, + json=block( + "dividends", + ["secid", "registryclosedate", "value", "currencyid"], + [["SBER", "2026-07-10", 34.84, "SUR"]], + ), + ) + ) + + +async def test_a_bond_run_writes_coupons_and_the_nominal_schedule(app, mock_http, run_sync): + instrument_id = await seed(AssetClass.bond, "RU000A") + mock_iss(mock_http) + + await run_sync(MoexPayoutsSource(), settings=Settings()) + + actions = await rows_of(CorporateAction, instrument_id=instrument_id) + assert {(a.kind, a.source, a.source_id) for a in actions} == { + (CorporateActionKind.coupon, "moex", "cpn:2027-05-05"), + (CorporateActionKind.coupon, "moex", "cpn:2028-05-05"), + } + schedule = sorted( + await rows_of(BondNominalSchedule, instrument_id=instrument_id), + key=lambda p: p.effective_date, + ) + assert [(p.effective_date, p.nominal) for p in schedule] == [ + (date(2027, 5, 5), D(750)), + (date(2028, 5, 5), D(0)), + ] + + +async def test_a_share_run_writes_the_register_dividend(app, mock_http, run_sync): + instrument_id = await seed(AssetClass.share, "SBER") + mock_iss(mock_http) + + await run_sync(MoexPayoutsSource(), settings=Settings()) + + actions = await rows_of(CorporateAction, instrument_id=instrument_id) + assert [(a.kind, a.source, a.amount_per_unit) for a in actions] == [ + (CorporateActionKind.dividend, "moex", D("34.84")) + ] + + +async def test_a_second_run_stores_no_duplicates(app, mock_http, run_sync): + instrument_id = await seed(AssetClass.bond, "RU000A") + mock_iss(mock_http) + + await run_sync(MoexPayoutsSource(), settings=Settings()) + await run_sync(MoexPayoutsSource(), settings=Settings()) + + assert len(await rows_of(CorporateAction, instrument_id=instrument_id)) == 2 + assert len(await rows_of(BondNominalSchedule, instrument_id=instrument_id)) == 2 + + +async def test_moex_wins_the_nominal_row_a_weaker_source_already_wrote(app, mock_http, run_sync): + """`bond_nominal_schedule` is keyed without `source`, so precedence is decided on write.""" + instrument_id = await seed(AssetClass.bond, "RU000A") + async with get_sessionmaker()() as session: + session.add( + BondNominalSchedule( + instrument_id=instrument_id, + effective_date=date(2027, 5, 5), + nominal=D(800), + currency="RUB", + source="tinvest", + ) + ) + await session.commit() + mock_iss(mock_http) + + await run_sync(MoexPayoutsSource(), settings=Settings()) + + schedule = await rows_of(BondNominalSchedule, instrument_id=instrument_id) + row = next(p for p in schedule if p.effective_date == date(2027, 5, 5)) + assert (row.nominal, row.source) == (D(750), "moex") diff --git a/backend/tests/sources/test_tinvest_events.py b/backend/tests/sources/test_tinvest_events.py new file mode 100644 index 0000000..20e0713 --- /dev/null +++ b/backend/tests/sources/test_tinvest_events.py @@ -0,0 +1,454 @@ +"""T-Invest payout feeds: what the three RPCs mean once they are flattened. + +The mapping tests are pure — the sync ones run against the real database with a stand-in +client, so no test here touches the network or needs a token. +""" + +from __future__ import annotations + +from datetime import UTC, date, datetime, timedelta +from decimal import Decimal +from typing import ClassVar + +import pytest +from sqlalchemy import select + +from factories import make_account, make_event +from fintracker.analytics import today_local +from fintracker.config import Settings +from fintracker.db import get_sessionmaker +from fintracker.models import ( + AccountKind, + AccountRole, + AssetClass, + BondNominalSchedule, + CorporateAction, + CorporateActionKind, + CorporateActionStatus, + EventKind, + Instrument, + RawTinvestEvent, +) +from fintracker.sources.tinvest import sync_events +from fintracker.sources.tinvest.client import BondCouponRow, BondEventRow, DividendRow +from fintracker.sources.tinvest.sync_events import ( + TinvestEventsSource, + coupon_payout, + dividend_payout, + msk_date, + nominal_schedule, + status_for, +) + +D = Decimal +TODAY = today_local() +PAST = TODAY - timedelta(days=30) +FUTURE = TODAY + timedelta(days=30) + + +def utc(day: date) -> datetime: + """How the API stamps a trading day: midnight UTC, i.e. 3 a.m. in Moscow.""" + return datetime(day.year, day.month, day.day, tzinfo=UTC) + + +def dividend(**over) -> DividendRow: + base = { + "instrument_uid": "uid-share", + "amount": D("45.24"), + "currency": "RUB", + "payment_date": utc(date(2026, 7, 24)), + "declared_date": utc(date(2026, 5, 30)), + "record_date": utc(date(2026, 7, 10)), + "last_buy_date": utc(date(2026, 7, 9)), + "dividend_type": "Regular Cash", + "regularity": "Annual", + "payload": {"dividend_net": {"value": "45.24", "currency": "rub"}}, + } + return DividendRow(**{**base, **over}) + + +def coupon(**over) -> BondCouponRow: + base = { + "instrument_uid": "uid-bond", + "coupon_number": 7, + "coupon_date": utc(date(2026, 11, 5)), + "fix_date": utc(date(2026, 11, 4)), + "pay_one_bond": D("34.90"), + "currency": "RUB", + "coupon_type": "COUPON_TYPE_CONSTANT", + "coupon_period": 182, + "payload": {"coupon_number": 7}, + } + return BondCouponRow(**{**base, **over}) + + +def redemption(day: date, amount: str, **over) -> BondEventRow: + base = { + "instrument_uid": "uid-bond", + "event_type": "EVENT_TYPE_MTY", + "event_number": 1, + "event_date": utc(day), + "fix_date": utc(day), + "pay_date": utc(day), + "pay_one_bond": D(amount), + "currency": "RUB", + "payload": {}, + } + return BondEventRow(**{**base, **over}) + + +# --- mapping ------------------------------------------------------------------------- + + +def test_a_midnight_utc_stamp_is_the_moscow_trading_day(): + """Read as UTC, every payout would move one day earlier than the exchange printed it.""" + assert msk_date(datetime(2026, 7, 10, 0, 0, tzinfo=UTC)) == date(2026, 7, 10) + assert msk_date(datetime(2026, 7, 9, 21, 30, tzinfo=UTC)) == date(2026, 7, 10) + assert msk_date(None) is None + + +def test_dividend_carries_every_date_the_feed_states(): + payout = dividend_payout(dividend(), instrument_id=7, today=TODAY) + + assert payout is not None + assert payout.kind is CorporateActionKind.dividend + assert payout.record_date == date(2026, 7, 10) + assert payout.pay_date == date(2026, 7, 24) + # last_buy_date is the last day a purchase still earns it — the ex-side date on offer + assert payout.ex_date == date(2026, 7, 9) + assert payout.amount_per_unit == D("45.24") + assert payout.currency == "RUB" + assert payout.source == "tinvest" + assert payout.source_id == "div:2026-07-10" + + +def test_dividend_without_any_date_is_dropped(): + assert ( + dividend_payout( + dividend(record_date=None, payment_date=None, declared_date=None), + instrument_id=7, + today=TODAY, + ) + is None + ) + + +def test_a_past_payout_is_paid_and_a_future_one_announced(): + assert status_for(PAST, TODAY) is CorporateActionStatus.paid + assert status_for(FUTURE, TODAY) is CorporateActionStatus.announced + assert status_for(None, TODAY) is CorporateActionStatus.announced + + +def test_coupon_maps_to_the_coupon_kind_keyed_on_its_number(): + warnings: list[str] = [] + payout = coupon_payout(coupon(), instrument_id=9, today=TODAY, warnings=warnings) + + assert payout is not None + assert payout.kind is CorporateActionKind.coupon + assert payout.pay_date == date(2026, 11, 5) + assert payout.record_date == date(2026, 11, 4) + assert payout.amount_per_unit == D("34.90") + assert payout.source_id == "cpn:7" + assert warnings == [] + + +def test_an_unknown_coupon_type_warns_instead_of_becoming_a_plain_coupon(): + warnings: list[str] = [] + payout = coupon_payout( + coupon(coupon_type="COUPON_TYPE_UNSPECIFIED"), + instrument_id=9, + today=TODAY, + warnings=warnings, + ) + + assert payout is None + assert len(warnings) == 1 + assert "COUPON_TYPE_UNSPECIFIED" in warnings[0] + + +def test_a_zero_coupon_is_not_a_payout(): + warnings: list[str] = [] + assert ( + coupon_payout( + coupon(pay_one_bond=D(0), coupon_type="COUPON_TYPE_DISCOUNT"), + instrument_id=9, + today=TODAY, + warnings=warnings, + ) + is None + ) + assert warnings == [] + + +def test_redemptions_run_the_nominal_down_to_zero(): + """An amortised bond repays the principal in slices; the nominal after each is what is left.""" + warnings: list[str] = [] + points = nominal_schedule( + [ + redemption(date(2027, 5, 5), "250"), + redemption(date(2026, 5, 5), "250"), # out of order on purpose + redemption(date(2028, 5, 5), "500"), + ], + instrument_id=9, + currency="RUB", + warnings=warnings, + ) + + assert [(p.effective_date, p.nominal) for p in points] == [ + (date(2026, 5, 5), D(750)), + (date(2027, 5, 5), D(500)), + (date(2028, 5, 5), D(0)), + ] + assert all(p.source == "tinvest" and p.currency == "RUB" for p in points) + assert warnings == [] + + +def test_a_bullet_bond_gets_a_single_point_at_maturity(): + points = nominal_schedule( + [redemption(date(2029, 3, 1), "1000")], instrument_id=9, currency="RUB", warnings=[] + ) + assert [(p.effective_date, p.nominal) for p in points] == [(date(2029, 3, 1), D(0))] + + +def test_an_unknown_bond_event_type_warns_and_is_skipped(): + warnings: list[str] = [] + points = nominal_schedule( + [redemption(date(2026, 5, 5), "250", event_type="EVENT_TYPE_CONV")], + instrument_id=9, + currency="RUB", + warnings=warnings, + ) + + assert points == [] + assert "EVENT_TYPE_CONV" in warnings[0] + + +def test_a_redemption_with_no_money_is_warned_about_not_read_as_zero(): + """A silent zero would shift every later nominal upward by the missing slice.""" + warnings: list[str] = [] + points = nominal_schedule( + [ + redemption(date(2026, 5, 5), "250", pay_one_bond=None), + redemption(date(2027, 5, 5), "750"), + ], + instrument_id=9, + currency="RUB", + warnings=warnings, + ) + + assert [p.nominal for p in points] == [D(0)] + assert "без суммы" in warnings[0] + + +def test_every_mapped_amount_is_a_decimal_never_a_float(): + payout = dividend_payout(dividend(), instrument_id=7, today=TODAY) + points = nominal_schedule( + [redemption(date(2026, 5, 5), "250"), redemption(date(2027, 5, 5), "750")], + instrument_id=9, + currency="RUB", + warnings=[], + ) + assert payout is not None + assert isinstance(payout.amount_per_unit, Decimal) + assert all(isinstance(p.nominal, Decimal) and not isinstance(p.nominal, float) for p in points) + + +# --- sync ---------------------------------------------------------------------------- + + +class FakeClient: + """Stands in for `TinvestClient`: same three coroutines, no gRPC and no token.""" + + calls: ClassVar[list[str]] = [] + + def __init__(self, token: str, **_: object) -> None: + self.token = token + + async def __aenter__(self) -> FakeClient: + return self + + async def __aexit__(self, *exc: object) -> None: + return None + + async def dividends(self, uid: str, **_: object) -> list[DividendRow]: + FakeClient.calls.append(f"dividends:{uid}") + return [dividend(instrument_uid=uid)] if uid == "uid-share" else [] + + async def bond_coupons(self, uid: str, **_: object) -> list[BondCouponRow]: + FakeClient.calls.append(f"coupons:{uid}") + return [ + coupon(instrument_uid=uid), + coupon(instrument_uid=uid, coupon_number=8, coupon_date=utc(date(2027, 5, 5))), + ] + + async def bond_events(self, uid: str, **_: object) -> list[BondEventRow]: + FakeClient.calls.append(f"events:{uid}") + return [redemption(date(2027, 5, 5), "250"), redemption(date(2028, 5, 5), "750")] + + +@pytest.fixture +def fake_client(monkeypatch): + FakeClient.calls = [] + monkeypatch.setattr(sync_events, "TinvestClient", FakeClient) + return FakeClient + + +async def seed(asset_class: AssetClass, uid: str, ticker: str) -> int: + """One instrument with a confirmed ledger event — which is what puts it in scope.""" + account_id = await make_account( + name="Брокерский", + kind=AccountKind.broker, + role=AccountRole.investment, + balance=None, + include_in_net_worth=False, + source="tinvest", + source_id=f"tinv-{ticker}", + ) + async with get_sessionmaker()() as session: + instrument = Instrument( + asset_class=asset_class, + tinvest_uid=uid, + ticker=ticker, + board="TQBR", + name=ticker, + currency="RUB", + ) + session.add(instrument) + await session.commit() + await session.refresh(instrument) + instrument_id = instrument.id + await make_event( + PAST, account_id=account_id, kind=EventKind.buy, instrument_id=instrument_id, quantity=10 + ) + return instrument_id + + +async def rows(model, **where): + async with get_sessionmaker()() as session: + stmt = select(model) + for column, value in where.items(): + stmt = stmt.where(getattr(model, column) == value) + return list((await session.execute(stmt)).scalars().all()) + + +def settings() -> Settings: + return Settings(tinvest_token="test-token") + + +async def test_a_share_run_writes_the_raw_row_and_the_corporate_action(app, fake_client, run_sync): + instrument_id = await seed(AssetClass.share, "uid-share", "SBER") + + result = await run_sync(TinvestEventsSource(), settings=settings()) + + assert result.counts["dividends"] == 1 + action = (await rows(CorporateAction, instrument_id=instrument_id))[0] + assert action.kind is CorporateActionKind.dividend + assert action.source == "tinvest" + assert action.source_id == "div:2026-07-10" + assert action.amount_per_unit == D("45.24") + assert action.record_date == date(2026, 7, 10) + raw = (await rows(RawTinvestEvent, instrument_uid="uid-share"))[0] + assert raw.kind == "dividend" + assert raw.payload["dividend_net"]["value"] == "45.24" + + +async def test_a_bond_run_writes_coupons_and_a_descending_nominal_schedule( + app, fake_client, run_sync +): + instrument_id = await seed(AssetClass.bond, "uid-bond", "RU000A") + + await run_sync(TinvestEventsSource(), settings=settings()) + + actions = await rows(CorporateAction, instrument_id=instrument_id) + assert {a.kind for a in actions} == {CorporateActionKind.coupon} + assert {a.source_id for a in actions} == {"cpn:7", "cpn:8"} + schedule = sorted( + await rows(BondNominalSchedule, instrument_id=instrument_id), + key=lambda p: p.effective_date, + ) + assert [(p.effective_date, p.nominal, p.source) for p in schedule] == [ + (date(2027, 5, 5), D(750), "tinvest"), + (date(2028, 5, 5), D(0), "tinvest"), + ] + + +async def test_a_bond_never_writes_an_amortization_row(app, fake_client, run_sync): + """`ledger/corporate_actions.py` owns that kind and prunes anything it did not derive.""" + instrument_id = await seed(AssetClass.bond, "uid-bond", "RU000A") + + await run_sync(TinvestEventsSource(), settings=settings()) + + actions = await rows(CorporateAction, instrument_id=instrument_id) + assert not any( + a.kind in (CorporateActionKind.amortization, CorporateActionKind.repayment) for a in actions + ) + + +async def test_a_share_is_never_asked_for_coupons(app, fake_client, run_sync): + await seed(AssetClass.share, "uid-share", "SBER") + + await run_sync(TinvestEventsSource(), settings=settings()) + + assert FakeClient.calls == ["dividends:uid-share"] + + +async def test_a_second_run_stores_no_duplicates(app, fake_client, run_sync): + instrument_id = await seed(AssetClass.bond, "uid-bond", "RU000A") + + await run_sync(TinvestEventsSource(), settings=settings()) + await run_sync(TinvestEventsSource(), settings=settings()) + + assert len(await rows(CorporateAction, instrument_id=instrument_id)) == 2 + assert len(await rows(BondNominalSchedule, instrument_id=instrument_id)) == 2 + assert len(await rows(RawTinvestEvent, instrument_uid="uid-bond")) == 4 + + +async def test_a_ledger_derived_paid_row_is_not_overwritten_by_the_feed(app, fake_client, run_sync): + """Money that arrived outranks an announcement — and the feed cannot even reach that row.""" + instrument_id = await seed(AssetClass.bond, "uid-bond", "RU000A") + async with get_sessionmaker()() as session: + session.add( + CorporateAction( + instrument_id=instrument_id, + kind=CorporateActionKind.amortization, + status=CorporateActionStatus.paid, + ex_date=date(2027, 5, 5), + pay_date=date(2027, 5, 5), + amount_per_unit=D("250"), + currency="RUB", + source="tinvest", + source_id="2027-05-05", + ) + ) + await session.commit() + + await run_sync(TinvestEventsSource(), settings=settings()) + + derived = [ + a + for a in await rows(CorporateAction, instrument_id=instrument_id) + if a.kind is CorporateActionKind.amortization + ] + assert len(derived) == 1 + assert derived[0].status is CorporateActionStatus.paid + assert derived[0].amount_per_unit == D("250") + + +async def test_an_instrument_outside_the_ledger_is_never_asked_about(app, fake_client, run_sync): + async with get_sessionmaker()() as session: + session.add( + Instrument( + asset_class=AssetClass.share, + tinvest_uid="uid-never-held", + ticker="GAZP", + board="TQBR", + name="GAZP", + currency="RUB", + ) + ) + await session.commit() + + result = await run_sync(TinvestEventsSource(), settings=settings()) + + assert FakeClient.calls == [] + assert result.changed is False diff --git a/docs/ai/phase4-contract.md b/docs/ai/phase4-contract.md new file mode 100644 index 0000000..c41253e --- /dev/null +++ b/docs/ai/phase4-contract.md @@ -0,0 +1,283 @@ +# Контракт API фазы 4 (доходы, ребалансировка, бенчмарки, цели, налоги) + +Единственный источник правды по формам запросов и ответов для бэкенда +(`api/routers/{income,goals,rebalance,tax,benchmarks}.py`, `api/schemas/*`) и для +Flutter-экранов. Пишется параллельно, поэтому имена полей и коды ошибок здесь важнее, чем +красота. Аналог `docs/ai/import-contract.md` для фазы 3. + +Общие правила проекта действуют без исключений: + +- деньги и количества — `Decimal` в Python и **строки** в JSON (`Money` / `MoneyOpt` из + `api/schemas/common.py`); никаких float. Ставки и доли — тоже строки-Decimal + (`"0.1234"` = 12,34 %), а не проценты и не float; +- даты — ISO-8601, таймстемпы — с таймзоной; ошибки — RFC 7807 (`Problem`); +- **енум `AssetClass` наружу не выставляется**: одно из его значений — `index`, а Dart-енум + не может иметь члена с таким именем. `asset_class`, `dimension`, `bucket`, `kind`, + `basis`, `period`, `status` — везде обычные строки; +- у каждого роута обязателен `name=`, иначе Dart-методы получат нечитаемые имена; +- `scope` — та же строка, что и в остальной аналитике: `all` | `account:` | + `portfolio:`; по умолчанию `all`. + +## 1. Доходы — `/income` + +### `GET /income/calendar` + +Параметры: `scope`, `date_from`, `date_to` (по умолчанию — от сегодня на 12 мес вперёд), +`include_paid` (bool, по умолчанию `false`). + +```jsonc +{ + "as_of": "2026-09-18", + "currency": "RUB", + "total_expected_rub": "14230.50", + "entries": [ + { + "instrument_id": 88, + "ticker": "SBER", + "name": "Сбербанк России", + "kind": "dividend", // dividend | coupon | amortization | repayment + "expected_date": "2026-10-12", + "record_date": "2026-10-09", + "qty": "20", + "per_unit": "34.84", + "amount": "696.80", + "currency": "RUB", + "amount_rub": "696.80", // null, если на дату нет курса + "basis": "announced", // schedule | announced | history | paid + "tax_withheld": null + } + ], + "by_basis": {"schedule": "8100.00", "announced": "4130.50", "history": "2000.00"} +} +``` + +**`basis` обязателен к показу.** Это не служебное поле: `schedule` — арифметика по +опубликованному графику, `announced` — объявленный эмитентом факт, `history` — +экстраполяция по последним 24 мес, которая может ошибаться на любую величину. Сложить их в +одно «ожидаемый доход» без разбивки — значит выдать догадку за прогноз. + +### `GET /income/history` + +Параметры: `scope`, `group` (`month`, по умолчанию), `date_from`, `date_to`, `kind`. + +```jsonc +{ + "rows": [ + {"month": "2026-08-01", "kind": "coupon", "currency": "RUB", + "amount": "1204.11", "amount_rub": "1204.11", "tax_withheld": "156.00", + "payment_count": 3} + ], + "totals": {"amount_rub": "24518.30", "tax_withheld_rub": "3187.00"} +} +``` + +### `GET /income/forecast` + +Параметры: `scope`, `months` (1..36, по умолчанию 12). + +```jsonc +{ + "months": [ + {"month": "2026-10-01", "amount_rub": "1830.20", + "by_basis": {"schedule": "1133.40", "announced": "696.80", "history": "0"}} + ], + "total_rub": "21960.00", + "annual_yield_on_value": "0.081", // ожидаемый доход / текущая стоимость; null, если нет оценки + "warnings": ["у 3 инструментов нет истории выплат — в прогноз не вошли"] +} +``` + +## 2. Ребалансировка — `/portfolios/{id}` + +### `GET /portfolios/{id}/targets` и `PUT /portfolios/{id}/targets` + +```jsonc +// PUT: тело — полный набор по одному измерению, частичное обновление не поддерживается +{ + "dimension": "asset_class", // asset_class | sector | country | currency + "targets": [ + {"bucket": "share", "target_weight": "0.60", "band": "0.05", "note": null}, + {"bucket": "bond", "target_weight": "0.30", "band": "0.05"}, + {"bucket": "cash", "target_weight": "0.10", "band": "0.02"} + ] +} +``` + +Ответ — сохранённый набор плюс `"weights_sum": "1.00"`. Сервер **не нормализует** веса: +сумма 0,9 — это ошибка пользователя, а не масштаб. При сумме, отличной от 1 более чем на +0,0001, — `422` с указанием фактической суммы. + +### `GET /portfolios/{id}/rebalance` + +Параметры: `dimension` (по умолчанию `asset_class`), `cash_available` (необязательно — +переопределяет остаток на счетах для what-if). + +```jsonc +{ + "portfolio_id": 1, + "dimension": "asset_class", + "as_of": "2026-09-18", + "total_value_rub": "1284300.00", + "cash_available_rub": "48120.87", + "buckets": [ + { + "bucket": "share", + "current_value_rub": "812000.00", + "current_weight": "0.632", + "target_weight": "0.60", + "drift": "0.032", // current - target, в долях + "within_band": true, // |drift| <= band ⇒ действий не предлагаем + "delta_value_rub": "-41420.00", + "trades": [ + { + "instrument_id": 88, "ticker": "SBER", "name": "Сбербанк России", + "action": "sell", // buy | sell + "suggested_qty": "150", // целые лоты; null, если нет цены + "lot": 10, + "price": "275.89", + "price_currency": "RUB", + "amount_rub": "41383.50", + "blocked_by_cash": false + } + ] + } + ], + "warnings": ["у 2 инструментов нет цены — в рекомендации не вошли"] +} +``` + +`suggested_qty` — **всегда целые лоты и всегда в пределах доступных денег**. Рекомендация, +которую нельзя исполнить, — не рекомендация. Если денег не хватает, количество урезается и +ставится `blocked_by_cash: true`. + +## 3. Бенчмарки — `/benchmarks` + +### `GET /benchmarks` / `POST /benchmarks` / `PATCH /benchmarks/{id}` / `DELETE` + +```jsonc +{"id": 1, "code": "MCFTR", "name": "MOEX Total Return", "kind": "total_return", + "source": "moex", "currency": "RUB", "is_default": true, "is_active": true, + "instrument_id": 412, "history_from": "2025-01-03", "history_to": "2026-09-17"} +``` + +### `GET /analytics/benchmarks` + +Параметры: `scope`, `period` (`1m 3m 6m ytd 1y 3y all`; можно повторять). + +```jsonc +{ + "rows": [ + { + "period": "1y", + "date_from": "2025-09-18", "date_to": "2026-09-18", + "portfolio_twr": "0.184", + "portfolio_twr_annualized": "0.184", + "portfolio_days_skipped": 3, + "benchmarks": [ + {"benchmark_id": 1, "code": "MCFTR", "kind": "total_return", + "twr": "0.121", "twr_annualized": "0.121", "days_skipped": 0, + "excess": "0.063"} // portfolio_twr - benchmark twr + ] + } + ] +} +``` + +**Сетка дат общая.** Бенчмарк, посчитанный по другому набору дней, — не сравнение; поэтому +`days_skipped` отдаётся с обеих сторон, и если он ненулевой, клиент обязан это показать. +Сравнивать портфель с ценовым индексом (`kind = "price"`) без пометки нельзя: IMOEX не +учитывает дивиденды и систематически занижает результат держателя. + +## 4. Цели — `/goals` + +CRUD: `GET /goals`, `POST /goals`, `PATCH /goals/{id}`, `DELETE /goals/{id}`. + +```jsonc +{"id": 3, "name": "Подушка", "scope": "account:12", "target_amount": "1000000", + "currency": "RUB", "target_date": "2028-01-01", "monthly_contribution": "30000", + "note": null, "archived": false} +``` + +### `GET /goals/{id}/progress` + +```jsonc +{ + "goal_id": 3, + "as_of": "2026-09-18", + "current_value_rub": "412800.00", + "target_amount_rub": "1000000.00", + "progress": "0.4128", + "projected_date": "2027-11-14", // null, если тренд не приводит к цели + "basis": "xirr", // xirr | contribution | none + "assumed_rate": "0.142", + "monthly_needed_rub": "42300.00", // сколько нужно докладывать, чтобы успеть к target_date + "on_track": false +} +``` + +`projected_date = null` — честный ответ «при текущем тренде цель не достигается». Дальняя +дата вместо null запрещена: она выглядит как ответ, не являясь им. + +## 5. Налоги — `/tax` + +### `GET /tax?year=2026` + +Параметры: `year` (по умолчанию текущий), `account_id` (необязательно). + +```jsonc +{ + "year": 2026, + "estimated": true, // ВСЕГДА true; авторитет — справка брокера + "tax_rate": "0.13", + "accounts": [ + { + "account_id": 12, "account_name": "ИИС Сбер", + "dividends_gross_rub": "12400.00", + "coupons_gross_rub": "8100.00", + "tax_withheld_rub": "2665.00", + "realized_gain_rub": "31200.00", + "realized_loss_rub": "-4100.00", + "ldv_exempt_rub": "12000.00", + "taxable_base_rub": "15100.00", + "estimated_tax_rub": "1963.00" + } + ], + "totals": { /* те же поля, суммарно */ }, + "disclaimer": "Оценка. Налоговый агент — брокер; сверяйтесь с его справкой." +} +``` + +### `GET /tax/lots?year=2026` + +Открытые лоты с датой, после которой продажа попадает под ЛДВ: + +```jsonc +{ + "lots": [ + {"lot_id": 812, "instrument_id": 88, "ticker": "SBER", "account_id": 12, + "open_date": "2024-03-14", "qty_remaining": "20", + "cost_rub": "4800.00", "market_value_rub": "5517.80", + "unrealized_gain_rub": "717.80", + "ldv_eligible": false, "ldv_date": "2027-03-14", "days_to_ldv": 177, + "tax_if_sold_now_rub": "93.31"} + ] +} +``` + +Это главный практический экран фазы: он показывает цену продажи бумаги **до** трёхлетней +отметки. `ldv_eligible` считается по ст. 219.1 и помечается оценкой — см. открытый вопрос 3 +плана (иностранные эмитенты и ИИС-3 сюда не заводим). + +## Что видит Flutter + +Новые экраны и маршруты: + +1. `/income` — календарь выплат (список по месяцам, `basis` виден на каждой строке), + вкладка «История» с помесячной таблицей и графиком, вкладка «Прогноз» на 12 мес. +2. `/rebalance` — целевые веса (редактирование) и рекомендации по текущему портфелю. +3. `/goals` — список целей с прогрессом; карточка цели с прогнозной датой. +4. `/tax` — сводка по году и список лотов с датой ЛДВ. +5. Бенчмарки — не отдельный экран, а блок сравнения на существующем «Портфель». + +После любого изменения целей, весов или бенчмарков клиент инвалидирует соответствующие +провайдеры: цифры пересчитываются на сервере при следующем refresh, а не в приложении.