feat(ledger): единый леджер событий и FIFO-лоты
Одна таблица event, куда маппится каждый брокерский источник: quantity знаковый по эффекту на позицию, amount — по эффекту на кэш, dedupe_key UNIQUE делает повторный импорт пустой операцией. Аналитика читает только confirmed. lots.apply() — чистая функция без сессии, rebuild.py её обвязка с БД. FIFO по (счёт, инструмент), как требует ст. 214.1 НК; комиссии капитализируются в покупку и вычитаются из продажи, НКД в себестоимость не входит — это деньги, авансированные продавцу и возвращаемые купоном. Короткие продажи — тоже лоты: продажа без остатка открывает короткий лот, покупка его закрывает, прибыль равна падению цены. В данных такое есть (TATN продан 22.08 и выкуплен 29.08); считать это ошибкой значило бы оставить фантомный длинный лот навсегда и потерять реализованную прибыль. Всё пересобирается с нуля на каждом refresh: объёмы личные, это секунды, зато исчезает целый класс багов расхождения инкремента с леджером.
This commit is contained in:
Dmitry
@@ -0,0 +1,444 @@
|
||||
"""леджер событий, лоты, цены и снапшоты (фаза 2)
|
||||
|
||||
Revision ID: 3b1431df06af
|
||||
Revises: b1d54240abb8
|
||||
Create Date: 2026-09-18 10:59:26.712697
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "3b1431df06af"
|
||||
down_revision: str | None = "b1d54240abb8"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"raw_tinvest_event",
|
||||
sa.Column("instrument_uid", sa.String(length=64), nullable=False),
|
||||
sa.Column("kind", sa.String(length=24), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=128), nullable=False),
|
||||
sa.Column("event_date", sa.Date(), nullable=True),
|
||||
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column(
|
||||
"fetched_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"instrument_uid", "kind", "source_id", name=op.f("pk_raw_tinvest_event")
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"raw_tinvest_instrument",
|
||||
sa.Column("uid", sa.String(length=64), nullable=False),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||
sa.Column("isin", sa.String(length=12), nullable=True),
|
||||
sa.Column("figi", sa.String(length=12), nullable=True),
|
||||
sa.Column("ticker", sa.String(length=32), nullable=True),
|
||||
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column(
|
||||
"fetched_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("uid", name=op.f("pk_raw_tinvest_instrument")),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_raw_tinvest_instrument_isin"), "raw_tinvest_instrument", ["isin"], unique=False
|
||||
)
|
||||
op.create_table(
|
||||
"raw_tinvest_operation",
|
||||
sa.Column("account_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("id", sa.String(length=64), nullable=False),
|
||||
sa.Column("operation_type", sa.String(length=64), nullable=False),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column(
|
||||
"fetched_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.PrimaryKeyConstraint("account_id", "id", name=op.f("pk_raw_tinvest_operation")),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_raw_tinvest_operation_operation_type"),
|
||||
"raw_tinvest_operation",
|
||||
["operation_type"],
|
||||
unique=False,
|
||||
)
|
||||
op.create_table(
|
||||
"raw_tinvest_snapshot",
|
||||
sa.Column("account_id", sa.String(length=64), nullable=False),
|
||||
sa.Column("kind", sa.String(length=16), nullable=False),
|
||||
sa.Column("captured_at", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"account_id", "kind", "captured_at", name=op.f("pk_raw_tinvest_snapshot")
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"cash_snapshot",
|
||||
sa.Column("account_id", sa.Integer(), nullable=False),
|
||||
sa.Column("currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("as_of", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("source", sa.String(length=16), nullable=False),
|
||||
sa.Column("balance", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("blocked", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["account_id"],
|
||||
["account.id"],
|
||||
name=op.f("fk_cash_snapshot_account_id_account"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"account_id", "currency", "as_of", "source", name=op.f("pk_cash_snapshot")
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"corporate_action",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=False),
|
||||
sa.Column(
|
||||
"kind",
|
||||
sa.Enum(
|
||||
"dividend",
|
||||
"coupon",
|
||||
"amortization",
|
||||
"repayment",
|
||||
"split",
|
||||
"offer",
|
||||
name="corporate_action_kind",
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum("forecast", "announced", "paid", "cancelled", name="corporate_action_status"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("record_date", sa.Date(), nullable=True),
|
||||
sa.Column("ex_date", sa.Date(), nullable=True),
|
||||
sa.Column("pay_date", sa.Date(), nullable=True),
|
||||
sa.Column("amount_per_unit", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("currency", sa.String(length=3), nullable=True),
|
||||
sa.Column("ratio", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("source", sa.String(length=16), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=128), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_corporate_action_instrument_id_instrument"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_corporate_action")),
|
||||
sa.UniqueConstraint(
|
||||
"instrument_id",
|
||||
"kind",
|
||||
"source",
|
||||
"source_id",
|
||||
name=op.f("uq_corporate_action_instrument_id_kind_source_source_id"),
|
||||
),
|
||||
)
|
||||
op.create_index("ix_corporate_action_pay_date", "corporate_action", ["pay_date"], unique=False)
|
||||
op.create_table(
|
||||
"event",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("account_id", sa.Integer(), nullable=False),
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=True),
|
||||
sa.Column(
|
||||
"kind",
|
||||
sa.Enum(
|
||||
"buy",
|
||||
"sell",
|
||||
"dividend",
|
||||
"coupon",
|
||||
"interest",
|
||||
"tax",
|
||||
"tax_refund",
|
||||
"commission",
|
||||
"deposit",
|
||||
"withdrawal",
|
||||
"transfer_in",
|
||||
"transfer_out",
|
||||
"split",
|
||||
"amortization",
|
||||
"repayment",
|
||||
"fx_exchange",
|
||||
"other",
|
||||
name="event_kind",
|
||||
),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum("confirmed", "pending", "shadow", "ignored", name="event_status"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("trade_date", sa.Date(), nullable=False),
|
||||
sa.Column("settle_date", sa.Date(), nullable=True),
|
||||
sa.Column("quantity", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("price", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("price_currency", sa.String(length=3), nullable=True),
|
||||
sa.Column("amount", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("fee", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("fee_currency", sa.String(length=3), nullable=True),
|
||||
sa.Column("tax", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("tax_currency", sa.String(length=3), nullable=True),
|
||||
sa.Column("accrued_interest", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("group_id", sa.UUID(), nullable=True),
|
||||
sa.Column("source", sa.String(length=32), nullable=False),
|
||||
sa.Column("source_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("raw_ref", sa.String(length=128), nullable=True),
|
||||
sa.Column("dedupe_key", sa.String(length=256), nullable=False),
|
||||
sa.Column("description", sa.Text(), nullable=True),
|
||||
sa.Column("meta", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["account_id"],
|
||||
["account.id"],
|
||||
name=op.f("fk_event_account_id_account"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_event_instrument_id_instrument"),
|
||||
ondelete="RESTRICT",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_event")),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_event_account_trade_date", "event", ["account_id", "trade_date"], unique=False
|
||||
)
|
||||
op.create_index(op.f("ix_event_group_id"), "event", ["group_id"], unique=False)
|
||||
op.create_index(
|
||||
"ix_event_instrument_trade_date", "event", ["instrument_id", "trade_date"], unique=False
|
||||
)
|
||||
op.create_index("uq_event_dedupe_key", "event", ["dedupe_key"], unique=True)
|
||||
op.create_table(
|
||||
"position_snapshot",
|
||||
sa.Column("account_id", sa.Integer(), nullable=False),
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=False),
|
||||
sa.Column("as_of", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("source", sa.String(length=16), nullable=False),
|
||||
sa.Column("qty", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("avg_price", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("market_value", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("currency", sa.String(length=3), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["account_id"],
|
||||
["account.id"],
|
||||
name=op.f("fk_position_snapshot_account_id_account"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_position_snapshot_instrument_id_instrument"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint(
|
||||
"account_id", "instrument_id", "as_of", "source", name=op.f("pk_position_snapshot")
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"price_daily",
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=False),
|
||||
sa.Column("d", sa.Date(), nullable=False),
|
||||
sa.Column("close", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("open", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("high", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("low", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("volume", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("source", sa.String(length=16), nullable=False),
|
||||
sa.Column("price_pct", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("accrued_interest", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_price_daily_instrument_id_instrument"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("instrument_id", "d", name=op.f("pk_price_daily")),
|
||||
)
|
||||
op.create_table(
|
||||
"price_last",
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=False),
|
||||
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
||||
sa.Column("price", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("source", sa.String(length=16), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_price_last_instrument_id_instrument"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("instrument_id", name=op.f("pk_price_last")),
|
||||
)
|
||||
op.create_table(
|
||||
"price_manual",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=False),
|
||||
sa.Column("d", sa.Date(), nullable=False),
|
||||
sa.Column("price", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("note", sa.String(length=256), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_price_manual_instrument_id_instrument"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_price_manual")),
|
||||
)
|
||||
op.create_table(
|
||||
"lot",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("account_id", sa.Integer(), nullable=False),
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=False),
|
||||
sa.Column("open_event_id", sa.Integer(), nullable=False),
|
||||
sa.Column("open_date", sa.Date(), nullable=False),
|
||||
sa.Column("qty_open", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("qty_remaining", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("cost_per_unit", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("cost_currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("cost_total_rub", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("closed_at", sa.Date(), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["account_id"],
|
||||
["account.id"],
|
||||
name=op.f("fk_lot_account_id_account"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_lot_instrument_id_instrument"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["open_event_id"],
|
||||
["event.id"],
|
||||
name=op.f("fk_lot_open_event_id_event"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_lot")),
|
||||
)
|
||||
op.create_index(
|
||||
"ix_lot_account_instrument", "lot", ["account_id", "instrument_id"], unique=False
|
||||
)
|
||||
op.create_table(
|
||||
"lot_disposal",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("lot_id", sa.Integer(), nullable=False),
|
||||
sa.Column("close_event_id", sa.Integer(), nullable=False),
|
||||
sa.Column("close_date", sa.Date(), nullable=False),
|
||||
sa.Column("qty", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("proceeds", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("proceeds_currency", sa.String(length=3), nullable=False),
|
||||
sa.Column("proceeds_rub", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("cost_rub", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("realized_pnl_native", sa.Numeric(precision=24, scale=10), nullable=False),
|
||||
sa.Column("realized_pnl_rub", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("holding_days", sa.Integer(), nullable=False),
|
||||
sa.Column("ldv_eligible", sa.Boolean(), nullable=False),
|
||||
sa.ForeignKeyConstraint(
|
||||
["close_event_id"],
|
||||
["event.id"],
|
||||
name=op.f("fk_lot_disposal_close_event_id_event"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["lot_id"], ["lot.id"], name=op.f("fk_lot_disposal_lot_id_lot"), ondelete="CASCADE"
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_lot_disposal")),
|
||||
)
|
||||
op.create_index("ix_lot_disposal_close_event", "lot_disposal", ["close_event_id"], unique=False)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index("ix_lot_disposal_close_event", table_name="lot_disposal")
|
||||
op.drop_table("lot_disposal")
|
||||
op.drop_index("ix_lot_account_instrument", table_name="lot")
|
||||
op.drop_table("lot")
|
||||
op.drop_table("price_manual")
|
||||
op.drop_table("price_last")
|
||||
op.drop_table("price_daily")
|
||||
op.drop_table("position_snapshot")
|
||||
op.drop_index("uq_event_dedupe_key", table_name="event")
|
||||
op.drop_index("ix_event_instrument_trade_date", table_name="event")
|
||||
op.drop_index(op.f("ix_event_group_id"), table_name="event")
|
||||
op.drop_index("ix_event_account_trade_date", table_name="event")
|
||||
op.drop_table("event")
|
||||
op.drop_index("ix_corporate_action_pay_date", table_name="corporate_action")
|
||||
op.drop_table("corporate_action")
|
||||
op.drop_table("cash_snapshot")
|
||||
op.drop_table("raw_tinvest_snapshot")
|
||||
op.drop_index(
|
||||
op.f("ix_raw_tinvest_operation_operation_type"), table_name="raw_tinvest_operation"
|
||||
)
|
||||
op.drop_table("raw_tinvest_operation")
|
||||
op.drop_index(op.f("ix_raw_tinvest_instrument_isin"), table_name="raw_tinvest_instrument")
|
||||
op.drop_table("raw_tinvest_instrument")
|
||||
op.drop_table("raw_tinvest_event")
|
||||
# ### end Alembic commands ###
|
||||
for enum_name in (
|
||||
"event_kind",
|
||||
"event_status",
|
||||
"corporate_action_kind",
|
||||
"corporate_action_status",
|
||||
):
|
||||
sa.Enum(name=enum_name).drop(op.get_bind(), checkfirst=True)
|
||||
@@ -0,0 +1 @@
|
||||
"""Derivations over the event ledger: FIFO lots, realised P&L, and source matching."""
|
||||
@@ -0,0 +1,299 @@
|
||||
"""FIFO lots and realised P&L (plan §1.5).
|
||||
|
||||
`apply()` is a pure function over a sequence of ledger events: no session, no I/O, so the
|
||||
rules below are testable on a handful of synthetic events. `rebuild_lots()` is the thin
|
||||
wrapper that loads from the database, runs it, and writes `lot` / `lot_disposal` back.
|
||||
|
||||
Rules, and why they are what they are:
|
||||
|
||||
* **FIFO per (account, instrument)**, as art. 214.1 NK requires for Russian tax. The same
|
||||
paper on two accounts is two independent queues — brokers compute tax per account.
|
||||
* **Commissions are capitalised into the purchase and deducted from the proceeds.** That
|
||||
is what makes realised P&L the number the tax authority expects, rather than a gross
|
||||
figure that overstates the gain.
|
||||
* **Accrued interest (НКД) is NOT part of the lot's cost.** It is money advanced to the
|
||||
seller and returned through the next coupon; folding it into cost would understate coupon
|
||||
income and overstate the gain on sale. It is tracked separately on the event.
|
||||
* **A split rewrites the open lots** — quantity times the ratio, cost per unit divided by
|
||||
it — so the total cost of a position never changes on a split.
|
||||
* **Amortisation reduces cost per unit**, since part of the principal has come back; the
|
||||
quantity is untouched. Without this, a fully amortised bond would show a phantom loss at
|
||||
repayment.
|
||||
* **Everything is rebuilt from scratch** on each refresh. The volumes are personal (a few
|
||||
thousand events), the recompute costs seconds, and it removes the entire class of bugs
|
||||
where an incremental update and the ledger disagree.
|
||||
|
||||
* **Short positions are lots too.** A sale with nothing to consume opens a SHORT lot, and a
|
||||
later purchase closes it — profit is the fall in price, the mirror of a long. Real data
|
||||
has these: TATN was sold on 22.08 and bought back on 29.08. Treating the sale as an error
|
||||
would leave a phantom long lot open forever and lose the realised gain.
|
||||
|
||||
A short lot still open at the end of the replay is the ambiguous case: it is either an open
|
||||
short or history that starts mid-way (a paper transferred in without a record). It is
|
||||
reported as a `Shortfall` for `metric_data_quality` to surface rather than guessed at.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import defaultdict
|
||||
from collections.abc import Sequence
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
|
||||
ZERO = Decimal(0)
|
||||
LDV_DAYS = 3 * 365
|
||||
"""Holding period qualifying for the long-term holding exemption (art. 219.1 NK)."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class LedgerEvent:
|
||||
"""What the lot engine needs from an `event` row — deliberately not the ORM object."""
|
||||
|
||||
id: int
|
||||
account_id: int
|
||||
instrument_id: int
|
||||
kind: EventKind
|
||||
trade_date: date
|
||||
quantity: Decimal | None
|
||||
"""Signed by position effect, as stored."""
|
||||
price: Decimal | None
|
||||
price_currency: str | None
|
||||
amount: Decimal
|
||||
currency: str
|
||||
fee: Decimal | None
|
||||
accrued_interest: Decimal | None = None
|
||||
ratio: Decimal | None = None
|
||||
"""Split ratio; only read for `stock_split` events."""
|
||||
exchange_traded: bool = True
|
||||
"""False for OTC papers, which do not qualify for the LDV exemption."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class OpenLot:
|
||||
account_id: int
|
||||
instrument_id: int
|
||||
open_event_id: int
|
||||
open_date: date
|
||||
qty_open: Decimal
|
||||
"""Always positive; `direction` says which way the position points."""
|
||||
qty_remaining: Decimal
|
||||
cost_per_unit: Decimal
|
||||
"""For a short lot this is the price it was SOLD at — what the buy-back is measured against."""
|
||||
cost_currency: str
|
||||
closed_at: date | None = None
|
||||
exchange_traded: bool = True
|
||||
direction: int = 1
|
||||
"""+1 long, -1 short."""
|
||||
|
||||
@property
|
||||
def signed_qty_remaining(self) -> Decimal:
|
||||
"""What the position actually is — negative while a short is open."""
|
||||
return self.qty_remaining * self.direction
|
||||
|
||||
|
||||
@dataclass
|
||||
class Disposal:
|
||||
lot: OpenLot
|
||||
close_event_id: int
|
||||
close_date: date
|
||||
qty: Decimal
|
||||
proceeds: Decimal
|
||||
proceeds_currency: str
|
||||
cost: Decimal
|
||||
"""Cost of the disposed quantity, in the lot's own currency."""
|
||||
realized_pnl_native: Decimal
|
||||
holding_days: int
|
||||
ldv_eligible: bool
|
||||
|
||||
|
||||
@dataclass
|
||||
class Shortfall:
|
||||
"""A disposal that found no lot to consume — history is incomplete, not wrong."""
|
||||
|
||||
account_id: int
|
||||
instrument_id: int
|
||||
close_event_id: int
|
||||
close_date: date
|
||||
qty: Decimal
|
||||
|
||||
|
||||
@dataclass
|
||||
class LotResult:
|
||||
lots: list[OpenLot] = field(default_factory=list)
|
||||
disposals: list[Disposal] = field(default_factory=list)
|
||||
shortfalls: list[Shortfall] = field(default_factory=list)
|
||||
|
||||
|
||||
def apply(events: Sequence[LedgerEvent]) -> LotResult:
|
||||
"""Replay events in chronological order into open lots and disposals."""
|
||||
result = LotResult()
|
||||
queues: dict[tuple[int, int], list[OpenLot]] = defaultdict(list)
|
||||
|
||||
for event in sorted(events, key=_order):
|
||||
key = (event.account_id, event.instrument_id)
|
||||
queue = queues[key]
|
||||
|
||||
if event.kind in {EventKind.buy, EventKind.transfer_in}:
|
||||
_trade(event, queue, result, direction=1)
|
||||
elif event.kind in {EventKind.sell, EventKind.transfer_out, EventKind.repayment}:
|
||||
_trade(event, queue, result, direction=-1)
|
||||
elif event.kind is EventKind.stock_split:
|
||||
_split(event, queue)
|
||||
elif event.kind is EventKind.amortization:
|
||||
_amortise(event, queue)
|
||||
|
||||
result.shortfalls = [
|
||||
Shortfall(
|
||||
account_id=lot.account_id,
|
||||
instrument_id=lot.instrument_id,
|
||||
close_event_id=lot.open_event_id,
|
||||
close_date=lot.open_date,
|
||||
qty=lot.qty_remaining,
|
||||
)
|
||||
for lot in result.lots
|
||||
if lot.direction == -1 and lot.qty_remaining > ZERO
|
||||
]
|
||||
return result
|
||||
|
||||
|
||||
def _order(event: LedgerEvent) -> tuple[date, int, int]:
|
||||
"""Chronological, and within a day: opens before closes, then by event id.
|
||||
|
||||
A same-day buy-then-sell must see the lot it created, and real feeds do not guarantee
|
||||
the order they hand same-day operations over.
|
||||
"""
|
||||
opens_first = 0 if event.kind in {EventKind.buy, EventKind.transfer_in} else 1
|
||||
return (event.trade_date, opens_first, event.id)
|
||||
|
||||
|
||||
def _trade(event: LedgerEvent, queue: list[OpenLot], result: LotResult, *, direction: int) -> None:
|
||||
"""One trade: first close whatever points the other way, then open a lot with the rest.
|
||||
|
||||
Buying closes short lots before opening a long one, and selling closes long lots before
|
||||
opening a short. That single rule covers ordinary round-trips and shorts alike.
|
||||
"""
|
||||
qty = abs(event.quantity or ZERO)
|
||||
if qty == ZERO:
|
||||
return
|
||||
per_unit = _per_unit(event, qty, direction)
|
||||
|
||||
remaining = _consume(event, queue, result, qty, per_unit, closing=-direction)
|
||||
if remaining > ZERO:
|
||||
lot = OpenLot(
|
||||
account_id=event.account_id,
|
||||
instrument_id=event.instrument_id,
|
||||
open_event_id=event.id,
|
||||
open_date=event.trade_date,
|
||||
qty_open=remaining,
|
||||
qty_remaining=remaining,
|
||||
cost_per_unit=per_unit,
|
||||
cost_currency=event.price_currency or event.currency,
|
||||
exchange_traded=event.exchange_traded,
|
||||
direction=direction,
|
||||
)
|
||||
queue.append(lot)
|
||||
result.lots.append(lot)
|
||||
|
||||
|
||||
def _per_unit(event: LedgerEvent, qty: Decimal, direction: int) -> Decimal:
|
||||
"""Value per unit, fees included on the side they fall.
|
||||
|
||||
A fee raises what a purchase costs and lowers what a sale nets — that asymmetry is why
|
||||
a flat round-trip still shows a loss, which is what the tax code expects.
|
||||
"""
|
||||
return _cash_basis(event, direction) / qty if qty else ZERO
|
||||
|
||||
|
||||
def _cash_basis(event: LedgerEvent, direction: int) -> Decimal:
|
||||
"""What the trade was worth per the cash that moved, accrued interest excluded.
|
||||
|
||||
`amount` is preferred because it is what actually hit the account; the price path is the
|
||||
fallback for feeds reporting no cash effect (a securities transfer). The fee is added on
|
||||
a buy and subtracted on a sale — T-Invest reports it as a separate operation, so
|
||||
`amount` does not already contain it either way.
|
||||
"""
|
||||
fee = abs(event.fee or ZERO) * direction
|
||||
if event.amount:
|
||||
gross = abs(event.amount) - abs(event.accrued_interest or ZERO)
|
||||
return gross + fee
|
||||
if event.price is not None:
|
||||
return event.price * abs(event.quantity or ZERO) + fee
|
||||
return ZERO
|
||||
|
||||
|
||||
def _consume(
|
||||
event: LedgerEvent,
|
||||
queue: list[OpenLot],
|
||||
result: LotResult,
|
||||
qty: Decimal,
|
||||
per_unit: Decimal,
|
||||
*,
|
||||
closing: int,
|
||||
) -> Decimal:
|
||||
"""Close lots pointing in the `closing` direction, FIFO; return what is left over."""
|
||||
remaining = qty
|
||||
for lot in queue:
|
||||
if remaining <= ZERO:
|
||||
break
|
||||
if lot.qty_remaining <= ZERO or lot.direction != closing:
|
||||
continue
|
||||
taken = min(lot.qty_remaining, remaining)
|
||||
lot.qty_remaining -= taken
|
||||
remaining -= taken
|
||||
if lot.qty_remaining == ZERO:
|
||||
lot.closed_at = event.trade_date
|
||||
|
||||
opened = lot.cost_per_unit * taken
|
||||
closed = per_unit * taken
|
||||
# long: bought at `opened`, sold at `closed`. short: sold at `opened`, bought at
|
||||
# `closed` — so the profit is the fall in price, and the roles simply swap.
|
||||
cost, proceeds = (opened, closed) if lot.direction == 1 else (closed, opened)
|
||||
holding_days = (event.trade_date - lot.open_date).days
|
||||
result.disposals.append(
|
||||
Disposal(
|
||||
lot=lot,
|
||||
close_event_id=event.id,
|
||||
close_date=event.trade_date,
|
||||
qty=taken,
|
||||
proceeds=proceeds,
|
||||
proceeds_currency=event.price_currency or event.currency,
|
||||
cost=cost,
|
||||
realized_pnl_native=proceeds - cost,
|
||||
holding_days=holding_days,
|
||||
ldv_eligible=(
|
||||
lot.direction == 1 and holding_days >= LDV_DAYS and lot.exchange_traded
|
||||
),
|
||||
)
|
||||
)
|
||||
return remaining
|
||||
|
||||
|
||||
def _split(event: LedgerEvent, queue: list[OpenLot]) -> None:
|
||||
"""Multiply quantities, divide cost per unit: the position's total cost is unchanged."""
|
||||
ratio = event.ratio
|
||||
if not ratio or ratio <= ZERO:
|
||||
return
|
||||
for lot in queue:
|
||||
if lot.qty_remaining <= ZERO:
|
||||
continue
|
||||
lot.qty_open *= ratio
|
||||
lot.qty_remaining *= ratio
|
||||
lot.cost_per_unit /= ratio
|
||||
|
||||
|
||||
def _amortise(event: LedgerEvent, queue: list[OpenLot]) -> None:
|
||||
"""Return of principal lowers what is still invested, proportionally across open lots."""
|
||||
paid = abs(event.amount)
|
||||
open_qty = sum((lot.qty_remaining for lot in queue if lot.qty_remaining > ZERO), start=ZERO)
|
||||
if paid == ZERO or open_qty == ZERO:
|
||||
return
|
||||
per_unit = paid / open_qty
|
||||
for lot in queue:
|
||||
if lot.qty_remaining <= ZERO:
|
||||
continue
|
||||
# never below zero: an amortisation bigger than the remaining cost means the cost
|
||||
# basis is already exhausted, and the excess is income, not a negative asset
|
||||
lot.cost_per_unit = max(lot.cost_per_unit - per_unit, ZERO)
|
||||
@@ -0,0 +1,193 @@
|
||||
"""Rebuild `lot` / `lot_disposal` from the event ledger (plan §1.5).
|
||||
|
||||
The engine in `lots.py` is pure; this module is the part that talks to the database: it
|
||||
loads confirmed events, runs the replay, converts to RUB at the rate of each leg's own date,
|
||||
and replaces the tables wholesale.
|
||||
|
||||
Currency handling follows the project rule: values are kept in the instrument's own currency
|
||||
and converted at the rate **of the date the money moved** — cost at the lot's open date,
|
||||
proceeds at the disposal date. A leg with no rate for its day yields NULL rather than a
|
||||
substituted number, and the gap is reported to data quality.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import delete, insert, select
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import FINDINGS
|
||||
from fintracker.ledger.lots import LedgerEvent, LotResult, apply
|
||||
from fintracker.models import Event, EventStatus, Instrument, Lot, LotDisposal
|
||||
from fintracker.models.ledger import POSITION_KINDS, EventKind
|
||||
from fintracker.models.pricing import CorporateAction, CorporateActionKind
|
||||
from fintracker.pricing.fx import FxTable
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
#: Kinds the engine reads: position moves plus the corporate actions that reshape lots.
|
||||
RELEVANT_KINDS = POSITION_KINDS | {EventKind.amortization}
|
||||
|
||||
#: Asset classes that can qualify for the long-term holding exemption.
|
||||
EXCHANGE_CLASSES = {"share", "bond", "etf", "fund"}
|
||||
|
||||
|
||||
async def rebuild_lots(session: AsyncSession) -> None:
|
||||
"""Replace `lot` and `lot_disposal` from scratch, and report what did not add up."""
|
||||
events = await _load_events(session)
|
||||
if not events:
|
||||
await session.execute(delete(LotDisposal))
|
||||
await session.execute(delete(Lot))
|
||||
return
|
||||
|
||||
result = apply(events)
|
||||
fx = await FxTable.load(session)
|
||||
|
||||
await session.execute(delete(LotDisposal))
|
||||
await session.execute(delete(Lot))
|
||||
|
||||
lot_ids: dict[int, int] = {}
|
||||
for lot in result.lots:
|
||||
cost_total = lot.cost_per_unit * lot.qty_open
|
||||
row = {
|
||||
"account_id": lot.account_id,
|
||||
"instrument_id": lot.instrument_id,
|
||||
"open_event_id": lot.open_event_id,
|
||||
"open_date": lot.open_date,
|
||||
# stored signed, so summing qty_remaining gives the position directly — a short
|
||||
# lot must subtract from it rather than look like a holding
|
||||
"qty_open": lot.qty_open * lot.direction,
|
||||
"qty_remaining": lot.signed_qty_remaining,
|
||||
"cost_per_unit": lot.cost_per_unit,
|
||||
"cost_currency": lot.cost_currency,
|
||||
"cost_total_rub": fx.to_rub(cost_total, lot.cost_currency, lot.open_date),
|
||||
"closed_at": lot.closed_at,
|
||||
}
|
||||
new_id = (await session.execute(insert(Lot).values(row).returning(Lot.id))).scalar_one()
|
||||
lot_ids[id(lot)] = new_id
|
||||
|
||||
disposal_rows = []
|
||||
for disposal in result.disposals:
|
||||
cost_rub = fx.to_rub(disposal.cost, disposal.lot.cost_currency, disposal.lot.open_date)
|
||||
proceeds_rub = fx.to_rub(disposal.proceeds, disposal.proceeds_currency, disposal.close_date)
|
||||
disposal_rows.append(
|
||||
{
|
||||
"lot_id": lot_ids[id(disposal.lot)],
|
||||
"close_event_id": disposal.close_event_id,
|
||||
"close_date": disposal.close_date,
|
||||
"qty": disposal.qty,
|
||||
"proceeds": disposal.proceeds,
|
||||
"proceeds_currency": disposal.proceeds_currency,
|
||||
"proceeds_rub": proceeds_rub,
|
||||
"cost_rub": cost_rub,
|
||||
"realized_pnl_native": disposal.realized_pnl_native,
|
||||
# both legs converted at their own dates; either missing makes it unknowable
|
||||
"realized_pnl_rub": (
|
||||
proceeds_rub - cost_rub
|
||||
if proceeds_rub is not None and cost_rub is not None
|
||||
else None
|
||||
),
|
||||
"holding_days": disposal.holding_days,
|
||||
"ldv_eligible": disposal.ldv_eligible,
|
||||
}
|
||||
)
|
||||
if disposal_rows:
|
||||
await session.execute(insert(LotDisposal), disposal_rows)
|
||||
|
||||
_report(result)
|
||||
log.info(
|
||||
"lots: %s lots, %s disposals, %s shortfalls",
|
||||
len(result.lots),
|
||||
len(result.disposals),
|
||||
len(result.shortfalls),
|
||||
)
|
||||
|
||||
|
||||
async def _load_events(session: AsyncSession) -> list[LedgerEvent]:
|
||||
"""Confirmed, instrument-bearing events, with the instrument facts the engine needs."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
Event.id,
|
||||
Event.account_id,
|
||||
Event.instrument_id,
|
||||
Event.kind,
|
||||
Event.trade_date,
|
||||
Event.quantity,
|
||||
Event.price,
|
||||
Event.price_currency,
|
||||
Event.amount,
|
||||
Event.currency,
|
||||
Event.fee,
|
||||
Event.accrued_interest,
|
||||
Instrument.asset_class,
|
||||
Instrument.board,
|
||||
)
|
||||
.join(Instrument, Instrument.id == Event.instrument_id)
|
||||
.where(
|
||||
Event.status == EventStatus.confirmed,
|
||||
Event.kind.in_(RELEVANT_KINDS),
|
||||
Event.instrument_id.is_not(None),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
|
||||
ratios = await _split_ratios(session)
|
||||
return [
|
||||
LedgerEvent(
|
||||
id=r.id,
|
||||
account_id=r.account_id,
|
||||
instrument_id=r.instrument_id,
|
||||
kind=r.kind,
|
||||
trade_date=r.trade_date,
|
||||
quantity=r.quantity,
|
||||
price=r.price,
|
||||
price_currency=r.price_currency,
|
||||
amount=r.amount or Decimal(0),
|
||||
currency=r.currency,
|
||||
fee=r.fee,
|
||||
accrued_interest=r.accrued_interest,
|
||||
ratio=ratios.get((r.instrument_id, r.trade_date)),
|
||||
exchange_traded=str(r.asset_class) in EXCHANGE_CLASSES and bool(r.board),
|
||||
)
|
||||
for r in rows
|
||||
]
|
||||
|
||||
|
||||
async def _split_ratios(session: AsyncSession) -> dict[tuple[int, object], Decimal]:
|
||||
"""Split ratios by (instrument, date) — the engine cannot infer them from an event."""
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(
|
||||
CorporateAction.instrument_id, CorporateAction.ex_date, CorporateAction.ratio
|
||||
).where(
|
||||
CorporateAction.kind == CorporateActionKind.stock_split,
|
||||
CorporateAction.ratio.is_not(None),
|
||||
)
|
||||
)
|
||||
).all()
|
||||
return {(r.instrument_id, r.ex_date): r.ratio for r in rows if r.ex_date}
|
||||
|
||||
|
||||
def _report(result: LotResult) -> None:
|
||||
"""Surface what the replay could not reconcile, aggregated per instrument."""
|
||||
if result.shortfalls:
|
||||
per_instrument: dict[int, Decimal] = {}
|
||||
for shortfall in result.shortfalls:
|
||||
per_instrument[shortfall.instrument_id] = (
|
||||
per_instrument.get(shortfall.instrument_id, Decimal(0)) + shortfall.qty
|
||||
)
|
||||
FINDINGS.add(
|
||||
"lot_shortfall",
|
||||
"warn",
|
||||
f"Продано больше, чем числится в лотах, по {len(per_instrument)} инструментам — "
|
||||
"история неполная (перевод бумаг или сделки до начала выгрузки)",
|
||||
count=len(result.shortfalls),
|
||||
ref={"instruments": sorted(per_instrument)},
|
||||
)
|
||||
|
||||
missing_fx = sum(1 for lot in result.lots if lot.cost_currency not in {"RUB", "rub"})
|
||||
if missing_fx:
|
||||
log.debug("lots: %s lots in a foreign currency", missing_fx)
|
||||
@@ -0,0 +1,182 @@
|
||||
"""The single event ledger and FIFO lots (plan §1.4, §1.5).
|
||||
|
||||
Every broker source maps into `event`: T-Invest operations, Sber/VTB report lines, manual
|
||||
entries. Day-to-day ZenMoney spending stays in `cash_txn` — it is not an investment event.
|
||||
|
||||
Sign conventions, so a reader never has to guess:
|
||||
|
||||
* `quantity` is signed by its effect on the POSITION: + into it, - out of it.
|
||||
* `amount` is signed by its effect on the ACCOUNT's cash: + received, - paid.
|
||||
A buy therefore has quantity > 0 and amount < 0.
|
||||
* `fee` and `tax` are stored positive (they are already reflected inside `amount`), so
|
||||
summing them never double-counts against the cash effect.
|
||||
|
||||
`dedupe_key` is what makes re-importing safe: every source builds a stable key from its own
|
||||
identifiers (a T-Invest operation id, a Sber deal number), and the unique index turns a
|
||||
repeated import into a no-op instead of a doubled ledger.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import uuid
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import ForeignKey, Index, Integer, String, Text
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class EventKind(enum.StrEnum):
|
||||
buy = "buy"
|
||||
sell = "sell"
|
||||
dividend = "dividend"
|
||||
coupon = "coupon"
|
||||
interest = "interest"
|
||||
tax = "tax"
|
||||
tax_refund = "tax_refund"
|
||||
commission = "commission"
|
||||
deposit = "deposit"
|
||||
withdrawal = "withdrawal"
|
||||
transfer_in = "transfer_in"
|
||||
transfer_out = "transfer_out"
|
||||
stock_split = "split"
|
||||
amortization = "amortization"
|
||||
repayment = "repayment"
|
||||
fx_exchange = "fx_exchange"
|
||||
other = "other"
|
||||
|
||||
|
||||
#: Kinds that move money across the portfolio boundary — the cash flows XIRR is computed on.
|
||||
#: Securities transferred in or out count too, valued at the market on their date.
|
||||
EXTERNAL_FLOW_KINDS = frozenset(
|
||||
{
|
||||
EventKind.deposit,
|
||||
EventKind.withdrawal,
|
||||
EventKind.transfer_in,
|
||||
EventKind.transfer_out,
|
||||
}
|
||||
)
|
||||
|
||||
#: Kinds that change a position's quantity, i.e. what `ledger/lots.py` consumes.
|
||||
POSITION_KINDS = frozenset(
|
||||
{
|
||||
EventKind.buy,
|
||||
EventKind.sell,
|
||||
EventKind.transfer_in,
|
||||
EventKind.transfer_out,
|
||||
EventKind.stock_split,
|
||||
EventKind.repayment,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class EventStatus(enum.StrEnum):
|
||||
confirmed = "confirmed"
|
||||
"""The truth for analytics — nothing else is read by a metric."""
|
||||
pending = "pending"
|
||||
"""Parsed but not committed yet (an import awaiting the user's confirmation)."""
|
||||
shadow = "shadow"
|
||||
"""A duplicate from a secondary source, kept for reconciliation only."""
|
||||
ignored = "ignored"
|
||||
|
||||
|
||||
class Event(TimestampMixin, Base):
|
||||
__tablename__ = "event"
|
||||
__table_args__ = (
|
||||
Index("uq_event_dedupe_key", "dedupe_key", unique=True),
|
||||
Index("ix_event_account_trade_date", "account_id", "trade_date"),
|
||||
Index("ix_event_instrument_trade_date", "instrument_id", "trade_date"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
account_id: Mapped[int] = mapped_column(ForeignKey("account.id", ondelete="CASCADE"))
|
||||
instrument_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="RESTRICT")
|
||||
)
|
||||
kind: Mapped[EventKind] = mapped_column(db_enum(EventKind, "event_kind"))
|
||||
status: Mapped[EventStatus] = mapped_column(
|
||||
db_enum(EventStatus, "event_status"), default=EventStatus.confirmed
|
||||
)
|
||||
|
||||
ts: Mapped[datetime]
|
||||
trade_date: Mapped[date]
|
||||
"""Trade date in MSK — the date every metric groups by, and the FX rate's date."""
|
||||
settle_date: Mapped[date | None]
|
||||
|
||||
quantity: Mapped[Decimal | None]
|
||||
"""Signed by position effect: + into the position, - out of it."""
|
||||
price: Mapped[Decimal | None]
|
||||
price_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
amount: Mapped[Decimal]
|
||||
"""Signed cash effect on the account: + received, - paid."""
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
fee: Mapped[Decimal | None]
|
||||
fee_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
tax: Mapped[Decimal | None]
|
||||
tax_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
accrued_interest: Mapped[Decimal | None]
|
||||
"""NKD paid (buy) or received (sell) on top of the bond's clean price."""
|
||||
|
||||
group_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), index=True)
|
||||
"""Ties the legs of one economic event together, e.g. both sides of an FX exchange."""
|
||||
|
||||
source: Mapped[str] = mapped_column(String(32))
|
||||
"""tinvest | report_sber | report_vtb | csv | manual"""
|
||||
source_id: Mapped[str | None] = mapped_column(String(128))
|
||||
raw_ref: Mapped[str | None] = mapped_column(String(128))
|
||||
dedupe_key: Mapped[str] = mapped_column(String(256))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
meta: Mapped[dict[str, Any] | None]
|
||||
|
||||
|
||||
class Lot(TimestampMixin, Base):
|
||||
"""One purchase kept open until sold, FIFO per (account, instrument) — art. 214.1 NK.
|
||||
|
||||
Rebuilt from scratch on every refresh by `ledger/lots.py`: the volumes are personal, so a
|
||||
full recompute costs seconds and removes a whole class of incremental-update bugs.
|
||||
"""
|
||||
|
||||
__tablename__ = "lot"
|
||||
__table_args__ = (Index("ix_lot_account_instrument", "account_id", "instrument_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
account_id: Mapped[int] = mapped_column(ForeignKey("account.id", ondelete="CASCADE"))
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
open_event_id: Mapped[int] = mapped_column(ForeignKey("event.id", ondelete="CASCADE"))
|
||||
open_date: Mapped[date]
|
||||
qty_open: Mapped[Decimal]
|
||||
qty_remaining: Mapped[Decimal]
|
||||
cost_per_unit: Mapped[Decimal]
|
||||
"""Purchase price plus capitalised fees, per unit, in `cost_currency`."""
|
||||
cost_currency: Mapped[str] = mapped_column(String(3))
|
||||
cost_total_rub: Mapped[Decimal | None]
|
||||
"""Cost in RUB at the CBR rate of `open_date`; NULL when that day has no rate."""
|
||||
closed_at: Mapped[date | None]
|
||||
|
||||
|
||||
class LotDisposal(Base):
|
||||
"""A sale (or repayment) consuming part of a lot: the realised-P&L and tax record."""
|
||||
|
||||
__tablename__ = "lot_disposal"
|
||||
__table_args__ = (Index("ix_lot_disposal_close_event", "close_event_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
lot_id: Mapped[int] = mapped_column(ForeignKey("lot.id", ondelete="CASCADE"))
|
||||
close_event_id: Mapped[int] = mapped_column(ForeignKey("event.id", ondelete="CASCADE"))
|
||||
close_date: Mapped[date]
|
||||
qty: Mapped[Decimal]
|
||||
proceeds: Mapped[Decimal]
|
||||
proceeds_currency: Mapped[str] = mapped_column(String(3))
|
||||
proceeds_rub: Mapped[Decimal | None]
|
||||
cost_rub: Mapped[Decimal | None]
|
||||
realized_pnl_native: Mapped[Decimal]
|
||||
realized_pnl_rub: Mapped[Decimal | None]
|
||||
"""NULL when either leg lacked an FX rate — never a substituted number."""
|
||||
holding_days: Mapped[int] = mapped_column(Integer)
|
||||
ldv_eligible: Mapped[bool]
|
||||
"""Held 3+ years on an exchange-traded instrument (art. 219.1 NK)."""
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Raw T-Invest payloads (plan §1.1): append-only, idempotent on the source's own ids.
|
||||
|
||||
Keeping the untouched payload means a mapping bug is fixable by re-deriving `event` from
|
||||
what is already stored, without asking the API for a full history again — which matters
|
||||
under a 200 req/min limit on Operations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base
|
||||
|
||||
|
||||
class RawTinvestOperation(Base):
|
||||
"""One `OperationItem` from GetOperationsByCursor, as returned."""
|
||||
|
||||
__tablename__ = "raw_tinvest_operation"
|
||||
|
||||
account_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
"""T-Invest account id (the broker's own string), not our `account.id`."""
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
operation_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
ts: Mapped[datetime]
|
||||
payload: Mapped[dict[str, Any]]
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawTinvestSnapshot(Base):
|
||||
"""A GetPortfolio / GetPositions capture — the reconciliation counterpart of the ledger."""
|
||||
|
||||
__tablename__ = "raw_tinvest_snapshot"
|
||||
|
||||
account_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
"""portfolio | positions"""
|
||||
captured_at: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
payload: Mapped[dict[str, Any]]
|
||||
|
||||
|
||||
class RawTinvestInstrument(Base):
|
||||
"""An instrument as the API describes it, keyed by its stable uid."""
|
||||
|
||||
__tablename__ = "raw_tinvest_instrument"
|
||||
|
||||
uid: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16))
|
||||
"""share | bond | etf | currency | future"""
|
||||
isin: Mapped[str | None] = mapped_column(String(12), index=True)
|
||||
figi: Mapped[str | None] = mapped_column(String(12))
|
||||
ticker: Mapped[str | None] = mapped_column(String(32))
|
||||
payload: Mapped[dict[str, Any]]
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawTinvestEvent(Base):
|
||||
"""Dividends, coupons, bond events and fundamentals, per instrument."""
|
||||
|
||||
__tablename__ = "raw_tinvest_event"
|
||||
|
||||
instrument_uid: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(24), primary_key=True)
|
||||
"""dividend | coupon | bond_event | fundamental"""
|
||||
source_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
event_date: Mapped[date | None]
|
||||
payload: Mapped[dict[str, Any]]
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
@@ -0,0 +1,243 @@
|
||||
"""FIFO engine rules, on synthetic events — no database, no source."""
|
||||
|
||||
from datetime import date, timedelta
|
||||
from decimal import Decimal
|
||||
|
||||
from fintracker.ledger.lots import LDV_DAYS, LedgerEvent, apply
|
||||
from fintracker.models.ledger import EventKind
|
||||
|
||||
D = Decimal
|
||||
ACC, INS = 1, 10
|
||||
_seq = iter(range(1, 10_000))
|
||||
|
||||
|
||||
def ev(kind: EventKind, day: str, *, qty=None, amount="0", price=None, **over) -> LedgerEvent:
|
||||
return LedgerEvent(
|
||||
id=over.pop("id", next(_seq)),
|
||||
account_id=over.pop("account_id", ACC),
|
||||
instrument_id=over.pop("instrument_id", INS),
|
||||
kind=kind,
|
||||
trade_date=date.fromisoformat(day),
|
||||
quantity=D(qty) if qty is not None else None,
|
||||
price=D(price) if price is not None else None,
|
||||
price_currency="RUB",
|
||||
amount=D(amount),
|
||||
currency="RUB",
|
||||
fee=D(over.pop("fee")) if "fee" in over else None,
|
||||
accrued_interest=D(over.pop("nkd")) if "nkd" in over else None,
|
||||
ratio=D(over.pop("ratio")) if "ratio" in over else None,
|
||||
exchange_traded=over.pop("exchange_traded", True),
|
||||
)
|
||||
|
||||
|
||||
def buy(day, qty, amount, **kw):
|
||||
return ev(EventKind.buy, day, qty=qty, amount=amount, **kw)
|
||||
|
||||
|
||||
def sell(day, qty, amount, **kw):
|
||||
return ev(EventKind.sell, day, qty=f"-{qty}", amount=amount, **kw)
|
||||
|
||||
|
||||
def test_fifo_consumes_the_oldest_lot_first():
|
||||
result = apply(
|
||||
[
|
||||
buy("2025-01-10", "10", "-1000"),
|
||||
buy("2025-02-10", "10", "-1500"),
|
||||
sell("2025-03-10", "10", "1400"),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(result.disposals) == 1
|
||||
disposal = result.disposals[0]
|
||||
assert disposal.lot.open_date == date(2025, 1, 10) # the January lot, not February
|
||||
assert disposal.cost == D(1000)
|
||||
assert disposal.realized_pnl_native == D(400)
|
||||
# the February lot is untouched
|
||||
assert [lot.qty_remaining for lot in result.lots] == [D(0), D(10)]
|
||||
|
||||
|
||||
def test_a_sale_spanning_two_lots_splits_into_two_disposals():
|
||||
result = apply(
|
||||
[
|
||||
buy("2025-01-10", "10", "-1000"),
|
||||
buy("2025-02-10", "10", "-2000"),
|
||||
sell("2025-03-10", "15", "2250"),
|
||||
]
|
||||
)
|
||||
|
||||
assert [d.qty for d in result.disposals] == [D(10), D(5)]
|
||||
# 10 @ cost 100 sold at 150 -> +500; 5 @ cost 200 sold at 150 -> -250
|
||||
assert [d.realized_pnl_native for d in result.disposals] == [D(500), D(-250)]
|
||||
assert sum(d.realized_pnl_native for d in result.disposals) == D(250)
|
||||
|
||||
|
||||
def test_fees_are_capitalised_into_cost_and_deducted_from_proceeds():
|
||||
"""The tax-relevant number: a buy costs more than the price, a sale nets less."""
|
||||
result = apply(
|
||||
[buy("2025-01-10", "10", "-1000", fee="10"), sell("2025-02-10", "10", "1000", fee="10")]
|
||||
)
|
||||
|
||||
disposal = result.disposals[0]
|
||||
assert disposal.cost == D(1010)
|
||||
assert disposal.proceeds == D(990)
|
||||
assert disposal.realized_pnl_native == D(-20) # a flat round-trip still loses both fees
|
||||
|
||||
|
||||
def test_accrued_interest_is_not_part_of_the_lot_cost():
|
||||
"""НКД is advanced to the seller and returned by the next coupon — not an investment."""
|
||||
result = apply([buy("2025-01-10", "10", "-1050", nkd="50")])
|
||||
|
||||
assert result.lots[0].cost_per_unit == D(100)
|
||||
|
||||
|
||||
def test_split_multiplies_quantity_and_divides_cost():
|
||||
result = apply(
|
||||
[buy("2025-01-10", "10", "-1000"), ev(EventKind.stock_split, "2025-02-01", ratio="10")]
|
||||
)
|
||||
|
||||
lot = result.lots[0]
|
||||
assert lot.qty_remaining == D(100)
|
||||
assert lot.cost_per_unit == D(10)
|
||||
assert lot.qty_remaining * lot.cost_per_unit == D(1000) # total cost unchanged
|
||||
|
||||
|
||||
def test_amortisation_lowers_cost_per_unit_without_touching_quantity():
|
||||
result = apply(
|
||||
[buy("2025-01-10", "10", "-10000"), ev(EventKind.amortization, "2025-06-01", amount="2000")]
|
||||
)
|
||||
|
||||
lot = result.lots[0]
|
||||
assert lot.qty_remaining == D(10)
|
||||
assert lot.cost_per_unit == D(800)
|
||||
|
||||
|
||||
def test_amortised_bond_repaid_at_par_shows_no_phantom_loss():
|
||||
"""Without amortisation lowering the basis, the repayment would look like a big loss."""
|
||||
result = apply(
|
||||
[
|
||||
buy("2025-01-10", "10", "-10000"),
|
||||
ev(EventKind.amortization, "2025-06-01", amount="2000"),
|
||||
ev(EventKind.repayment, "2025-12-01", qty="-10", amount="8000"),
|
||||
]
|
||||
)
|
||||
|
||||
assert result.disposals[0].realized_pnl_native == D(0)
|
||||
|
||||
|
||||
def test_ldv_marks_only_holdings_of_three_years_or_more():
|
||||
open_day = date(2021, 1, 10)
|
||||
long_enough = open_day + timedelta(days=LDV_DAYS)
|
||||
result = apply(
|
||||
[buy(open_day.isoformat(), "10", "-1000"), sell(long_enough.isoformat(), "10", "1500")]
|
||||
)
|
||||
|
||||
assert result.disposals[0].ldv_eligible is True
|
||||
|
||||
|
||||
def test_ldv_does_not_apply_a_day_early():
|
||||
open_day = date(2021, 1, 10)
|
||||
too_soon = open_day + timedelta(days=LDV_DAYS - 1)
|
||||
result = apply(
|
||||
[buy(open_day.isoformat(), "10", "-1000"), sell(too_soon.isoformat(), "10", "1500")]
|
||||
)
|
||||
|
||||
assert result.disposals[0].ldv_eligible is False
|
||||
|
||||
|
||||
def test_ldv_does_not_apply_to_otc_papers():
|
||||
open_day = date(2021, 1, 10)
|
||||
long_enough = open_day + timedelta(days=LDV_DAYS)
|
||||
result = apply(
|
||||
[
|
||||
buy(open_day.isoformat(), "10", "-1000", exchange_traded=False),
|
||||
sell(long_enough.isoformat(), "10", "1500"),
|
||||
]
|
||||
)
|
||||
|
||||
assert result.disposals[0].ldv_eligible is False
|
||||
|
||||
|
||||
def test_selling_more_than_held_reports_a_shortfall_instead_of_inventing_a_lot():
|
||||
result = apply([buy("2025-01-10", "5", "-500"), sell("2025-02-10", "8", "800")])
|
||||
|
||||
assert result.disposals[0].qty == D(5)
|
||||
assert [s.qty for s in result.shortfalls] == [D(3)]
|
||||
|
||||
|
||||
def test_a_same_day_buy_is_available_to_a_same_day_sale():
|
||||
"""Feeds do not order same-day operations; the engine must not depend on luck."""
|
||||
sale = sell("2025-01-10", "10", "1200", id=1)
|
||||
purchase = buy("2025-01-10", "10", "-1000", id=2)
|
||||
|
||||
result = apply([sale, purchase])
|
||||
|
||||
assert not result.shortfalls
|
||||
assert result.disposals[0].realized_pnl_native == D(200)
|
||||
|
||||
|
||||
def test_accounts_keep_independent_queues():
|
||||
"""Tax is computed per account, so the same paper elsewhere must not be consumed."""
|
||||
result = apply(
|
||||
[
|
||||
buy("2025-01-10", "10", "-1000", account_id=1),
|
||||
buy("2025-01-11", "10", "-2000", account_id=2),
|
||||
sell("2025-02-10", "10", "1500", account_id=2),
|
||||
]
|
||||
)
|
||||
|
||||
disposal = result.disposals[0]
|
||||
assert disposal.lot.account_id == 2
|
||||
assert disposal.cost == D(2000) # account 2's own lot, not account 1's cheaper one
|
||||
|
||||
|
||||
def test_transfer_in_opens_a_lot_and_transfer_out_closes_one():
|
||||
result = apply(
|
||||
[
|
||||
ev(EventKind.transfer_in, "2025-01-10", qty="10", amount="0", price="100"),
|
||||
ev(EventKind.transfer_out, "2025-02-10", qty="-4", amount="0", price="120"),
|
||||
]
|
||||
)
|
||||
|
||||
assert result.lots[0].cost_per_unit == D(100)
|
||||
assert result.disposals[0].qty == D(4)
|
||||
|
||||
|
||||
def test_a_short_round_trip_realises_the_fall_in_price():
|
||||
"""Real case (TATN): sold on 22.08, bought back on 29.08 — profit is the price drop."""
|
||||
result = apply(
|
||||
[
|
||||
sell("2025-08-22", "10", "6751"),
|
||||
buy("2025-08-29", "10", "-6505"),
|
||||
]
|
||||
)
|
||||
|
||||
assert not result.shortfalls
|
||||
assert result.disposals[0].realized_pnl_native == D(246)
|
||||
assert sum(lot.signed_qty_remaining for lot in result.lots) == D(0)
|
||||
|
||||
|
||||
def test_an_open_short_is_reported_as_a_shortfall():
|
||||
"""Either an open short or history starting mid-way — both need a human to look."""
|
||||
result = apply([sell("2025-08-22", "10", "6751")])
|
||||
|
||||
assert [s.qty for s in result.shortfalls] == [D(10)]
|
||||
assert result.lots[0].signed_qty_remaining == D(-10)
|
||||
|
||||
|
||||
def test_a_short_is_never_ldv_eligible():
|
||||
result = apply(
|
||||
[
|
||||
sell("2021-01-10", "10", "1500"),
|
||||
buy((date(2021, 1, 10) + timedelta(days=LDV_DAYS)).isoformat(), "10", "-1000"),
|
||||
]
|
||||
)
|
||||
|
||||
assert result.disposals[0].ldv_eligible is False
|
||||
|
||||
|
||||
def test_selling_past_a_long_position_opens_a_short_for_the_excess():
|
||||
result = apply([buy("2025-01-10", "5", "-500"), sell("2025-02-10", "8", "800")])
|
||||
|
||||
assert result.disposals[0].qty == D(5)
|
||||
assert [s.qty for s in result.shortfalls] == [D(3)]
|
||||
assert sum(lot.signed_qty_remaining for lot in result.lots) == D(-3)
|
||||
Reference in New Issue
Block a user