feat(ledger): единый леджер событий и FIFO-лоты
Одна таблица event, куда маппится каждый брокерский источник: quantity знаковый по эффекту на позицию, amount — по эффекту на кэш, dedupe_key UNIQUE делает повторный импорт пустой операцией. Аналитика читает только confirmed. lots.apply() — чистая функция без сессии, rebuild.py её обвязка с БД. FIFO по (счёт, инструмент), как требует ст. 214.1 НК; комиссии капитализируются в покупку и вычитаются из продажи, НКД в себестоимость не входит — это деньги, авансированные продавцу и возвращаемые купоном. Короткие продажи — тоже лоты: продажа без остатка открывает короткий лот, покупка его закрывает, прибыль равна падению цены. В данных такое есть (TATN продан 22.08 и выкуплен 29.08); считать это ошибкой значило бы оставить фантомный длинный лот навсегда и потерять реализованную прибыль. Всё пересобирается с нуля на каждом refresh: объёмы личные, это секунды, зато исчезает целый класс багов расхождения инкремента с леджером.
This commit is contained in:
Dmitry
@@ -0,0 +1,182 @@
|
||||
"""The single event ledger and FIFO lots (plan §1.4, §1.5).
|
||||
|
||||
Every broker source maps into `event`: T-Invest operations, Sber/VTB report lines, manual
|
||||
entries. Day-to-day ZenMoney spending stays in `cash_txn` — it is not an investment event.
|
||||
|
||||
Sign conventions, so a reader never has to guess:
|
||||
|
||||
* `quantity` is signed by its effect on the POSITION: + into it, - out of it.
|
||||
* `amount` is signed by its effect on the ACCOUNT's cash: + received, - paid.
|
||||
A buy therefore has quantity > 0 and amount < 0.
|
||||
* `fee` and `tax` are stored positive (they are already reflected inside `amount`), so
|
||||
summing them never double-counts against the cash effect.
|
||||
|
||||
`dedupe_key` is what makes re-importing safe: every source builds a stable key from its own
|
||||
identifiers (a T-Invest operation id, a Sber deal number), and the unique index turns a
|
||||
repeated import into a no-op instead of a doubled ledger.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import uuid
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import ForeignKey, Index, Integer, String, Text
|
||||
from sqlalchemy.dialects.postgresql import UUID
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class EventKind(enum.StrEnum):
|
||||
buy = "buy"
|
||||
sell = "sell"
|
||||
dividend = "dividend"
|
||||
coupon = "coupon"
|
||||
interest = "interest"
|
||||
tax = "tax"
|
||||
tax_refund = "tax_refund"
|
||||
commission = "commission"
|
||||
deposit = "deposit"
|
||||
withdrawal = "withdrawal"
|
||||
transfer_in = "transfer_in"
|
||||
transfer_out = "transfer_out"
|
||||
stock_split = "split"
|
||||
amortization = "amortization"
|
||||
repayment = "repayment"
|
||||
fx_exchange = "fx_exchange"
|
||||
other = "other"
|
||||
|
||||
|
||||
#: Kinds that move money across the portfolio boundary — the cash flows XIRR is computed on.
|
||||
#: Securities transferred in or out count too, valued at the market on their date.
|
||||
EXTERNAL_FLOW_KINDS = frozenset(
|
||||
{
|
||||
EventKind.deposit,
|
||||
EventKind.withdrawal,
|
||||
EventKind.transfer_in,
|
||||
EventKind.transfer_out,
|
||||
}
|
||||
)
|
||||
|
||||
#: Kinds that change a position's quantity, i.e. what `ledger/lots.py` consumes.
|
||||
POSITION_KINDS = frozenset(
|
||||
{
|
||||
EventKind.buy,
|
||||
EventKind.sell,
|
||||
EventKind.transfer_in,
|
||||
EventKind.transfer_out,
|
||||
EventKind.stock_split,
|
||||
EventKind.repayment,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
class EventStatus(enum.StrEnum):
|
||||
confirmed = "confirmed"
|
||||
"""The truth for analytics — nothing else is read by a metric."""
|
||||
pending = "pending"
|
||||
"""Parsed but not committed yet (an import awaiting the user's confirmation)."""
|
||||
shadow = "shadow"
|
||||
"""A duplicate from a secondary source, kept for reconciliation only."""
|
||||
ignored = "ignored"
|
||||
|
||||
|
||||
class Event(TimestampMixin, Base):
|
||||
__tablename__ = "event"
|
||||
__table_args__ = (
|
||||
Index("uq_event_dedupe_key", "dedupe_key", unique=True),
|
||||
Index("ix_event_account_trade_date", "account_id", "trade_date"),
|
||||
Index("ix_event_instrument_trade_date", "instrument_id", "trade_date"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
account_id: Mapped[int] = mapped_column(ForeignKey("account.id", ondelete="CASCADE"))
|
||||
instrument_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="RESTRICT")
|
||||
)
|
||||
kind: Mapped[EventKind] = mapped_column(db_enum(EventKind, "event_kind"))
|
||||
status: Mapped[EventStatus] = mapped_column(
|
||||
db_enum(EventStatus, "event_status"), default=EventStatus.confirmed
|
||||
)
|
||||
|
||||
ts: Mapped[datetime]
|
||||
trade_date: Mapped[date]
|
||||
"""Trade date in MSK — the date every metric groups by, and the FX rate's date."""
|
||||
settle_date: Mapped[date | None]
|
||||
|
||||
quantity: Mapped[Decimal | None]
|
||||
"""Signed by position effect: + into the position, - out of it."""
|
||||
price: Mapped[Decimal | None]
|
||||
price_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
amount: Mapped[Decimal]
|
||||
"""Signed cash effect on the account: + received, - paid."""
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
fee: Mapped[Decimal | None]
|
||||
fee_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
tax: Mapped[Decimal | None]
|
||||
tax_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
accrued_interest: Mapped[Decimal | None]
|
||||
"""NKD paid (buy) or received (sell) on top of the bond's clean price."""
|
||||
|
||||
group_id: Mapped[uuid.UUID | None] = mapped_column(UUID(as_uuid=True), index=True)
|
||||
"""Ties the legs of one economic event together, e.g. both sides of an FX exchange."""
|
||||
|
||||
source: Mapped[str] = mapped_column(String(32))
|
||||
"""tinvest | report_sber | report_vtb | csv | manual"""
|
||||
source_id: Mapped[str | None] = mapped_column(String(128))
|
||||
raw_ref: Mapped[str | None] = mapped_column(String(128))
|
||||
dedupe_key: Mapped[str] = mapped_column(String(256))
|
||||
description: Mapped[str | None] = mapped_column(Text)
|
||||
meta: Mapped[dict[str, Any] | None]
|
||||
|
||||
|
||||
class Lot(TimestampMixin, Base):
|
||||
"""One purchase kept open until sold, FIFO per (account, instrument) — art. 214.1 NK.
|
||||
|
||||
Rebuilt from scratch on every refresh by `ledger/lots.py`: the volumes are personal, so a
|
||||
full recompute costs seconds and removes a whole class of incremental-update bugs.
|
||||
"""
|
||||
|
||||
__tablename__ = "lot"
|
||||
__table_args__ = (Index("ix_lot_account_instrument", "account_id", "instrument_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
account_id: Mapped[int] = mapped_column(ForeignKey("account.id", ondelete="CASCADE"))
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
open_event_id: Mapped[int] = mapped_column(ForeignKey("event.id", ondelete="CASCADE"))
|
||||
open_date: Mapped[date]
|
||||
qty_open: Mapped[Decimal]
|
||||
qty_remaining: Mapped[Decimal]
|
||||
cost_per_unit: Mapped[Decimal]
|
||||
"""Purchase price plus capitalised fees, per unit, in `cost_currency`."""
|
||||
cost_currency: Mapped[str] = mapped_column(String(3))
|
||||
cost_total_rub: Mapped[Decimal | None]
|
||||
"""Cost in RUB at the CBR rate of `open_date`; NULL when that day has no rate."""
|
||||
closed_at: Mapped[date | None]
|
||||
|
||||
|
||||
class LotDisposal(Base):
|
||||
"""A sale (or repayment) consuming part of a lot: the realised-P&L and tax record."""
|
||||
|
||||
__tablename__ = "lot_disposal"
|
||||
__table_args__ = (Index("ix_lot_disposal_close_event", "close_event_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
lot_id: Mapped[int] = mapped_column(ForeignKey("lot.id", ondelete="CASCADE"))
|
||||
close_event_id: Mapped[int] = mapped_column(ForeignKey("event.id", ondelete="CASCADE"))
|
||||
close_date: Mapped[date]
|
||||
qty: Mapped[Decimal]
|
||||
proceeds: Mapped[Decimal]
|
||||
proceeds_currency: Mapped[str] = mapped_column(String(3))
|
||||
proceeds_rub: Mapped[Decimal | None]
|
||||
cost_rub: Mapped[Decimal | None]
|
||||
realized_pnl_native: Mapped[Decimal]
|
||||
realized_pnl_rub: Mapped[Decimal | None]
|
||||
"""NULL when either leg lacked an FX rate — never a substituted number."""
|
||||
holding_days: Mapped[int] = mapped_column(Integer)
|
||||
ldv_eligible: Mapped[bool]
|
||||
"""Held 3+ years on an exchange-traded instrument (art. 219.1 NK)."""
|
||||
@@ -0,0 +1,71 @@
|
||||
"""Raw T-Invest payloads (plan §1.1): append-only, idempotent on the source's own ids.
|
||||
|
||||
Keeping the untouched payload means a mapping bug is fixable by re-deriving `event` from
|
||||
what is already stored, without asking the API for a full history again — which matters
|
||||
under a 200 req/min limit on Operations.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base
|
||||
|
||||
|
||||
class RawTinvestOperation(Base):
|
||||
"""One `OperationItem` from GetOperationsByCursor, as returned."""
|
||||
|
||||
__tablename__ = "raw_tinvest_operation"
|
||||
|
||||
account_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
"""T-Invest account id (the broker's own string), not our `account.id`."""
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
operation_type: Mapped[str] = mapped_column(String(64), index=True)
|
||||
ts: Mapped[datetime]
|
||||
payload: Mapped[dict[str, Any]]
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawTinvestSnapshot(Base):
|
||||
"""A GetPortfolio / GetPositions capture — the reconciliation counterpart of the ledger."""
|
||||
|
||||
__tablename__ = "raw_tinvest_snapshot"
|
||||
|
||||
account_id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
"""portfolio | positions"""
|
||||
captured_at: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
payload: Mapped[dict[str, Any]]
|
||||
|
||||
|
||||
class RawTinvestInstrument(Base):
|
||||
"""An instrument as the API describes it, keyed by its stable uid."""
|
||||
|
||||
__tablename__ = "raw_tinvest_instrument"
|
||||
|
||||
uid: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(16))
|
||||
"""share | bond | etf | currency | future"""
|
||||
isin: Mapped[str | None] = mapped_column(String(12), index=True)
|
||||
figi: Mapped[str | None] = mapped_column(String(12))
|
||||
ticker: Mapped[str | None] = mapped_column(String(32))
|
||||
payload: Mapped[dict[str, Any]]
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawTinvestEvent(Base):
|
||||
"""Dividends, coupons, bond events and fundamentals, per instrument."""
|
||||
|
||||
__tablename__ = "raw_tinvest_event"
|
||||
|
||||
instrument_uid: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
kind: Mapped[str] = mapped_column(String(24), primary_key=True)
|
||||
"""dividend | coupon | bond_event | fundamental"""
|
||||
source_id: Mapped[str] = mapped_column(String(128), primary_key=True)
|
||||
event_date: Mapped[date | None]
|
||||
payload: Mapped[dict[str, Any]]
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
Reference in New Issue
Block a user