feat(ledger): импорт отчётов в леджер — приём, дедупликация, pending_instrument

Поток: upload -> raw_report_file (sha256 UNIQUE) -> parse -> raw_report_line, событий в леджере ещё нет -> preview -> POST /imports/{id}/commit -> ledger/ingest.py резолвит инструмент, считает dedupe_key, пишет event confirmed | shadow (plan §1.6 B) — сравнивая account.primary_event_source с источником отчёта, а не гадая. Нерезолвленный инструмент ждёт в pending_instrument, никогда не угадывается; POST /instruments/pending/{id}/resolve привязывает и пересобирает лоты. ledger/dedupe.py — shadow-матчинг случая B двумя проходами (точная дата, затем ±1 рабочий день, жадно 1:1, |price| ±0,5 %). Шаги shadow_dedupe и report_reconcile зарегистрированы перед quality: оба говорят через FINDINGS. /instruments/pending регистрируется в app.py ДО routers/instruments.py: FastAPI сопоставляет маршруты по порядку, и /instruments/{id} с типом int отвечает 422 на нечисловой сегмент, а не проваливается дальше. Контракт — docs/ai/import-contract.md, общий для бэкенда и Flutter.
2026-09-19 10:40:04 +03:00
parent 2e742a093b
commit ff3b76871d
16 changed files with 4484 additions and 0 deletions
@@ -0,0 +1,139 @@
+"""Uploaded broker reports: the raw tier and what they could not resolve (plan §1.1, §1.3).
+
+An import is two-phase on purpose. `raw_report_file` records the bytes' sha256 the moment
+they arrive, so re-uploading the same file is a no-op rather than a doubled ledger — the
+unique index is the whole mechanism, and it works even when the file is renamed (VTB names
+its exports by GUID). `raw_report_line` keeps every parsed row as JSONB before normalisation,
+which is what makes a mapping bug diagnosable months later: the ledger can be rebuilt from
+these rows without asking the user for the file again.
+
+`pending_instrument` exists because guessing is worse than waiting. A report prints
+«Первая-ВечныйПортф БПИФ / STME» and nothing else; matching that to an instrument by name
+similarity would silently attach trades to the wrong paper, and a wrong lot is far more
+expensive to notice than an unresolved one. The row parks the unknown, the events that
+reference it stay `pending`, and the user confirms the instrument in the UI.
+"""
+
+from __future__ import annotations
+
+import enum
+from datetime import date, datetime
+from decimal import Decimal
+from typing import Any
+
+from sqlalchemy import BigInteger, ForeignKey, Index, Integer, String, Text, UniqueConstraint
+from sqlalchemy.orm import Mapped, mapped_column
+
+from fintracker.db.base import Base, TimestampMixin, db_enum
+
+
+class ReportParseStatus(enum.StrEnum):
+    uploaded = "uploaded"
+    """Bytes stored, not parsed yet."""
+    parsed = "parsed"
+    """Parsed into `raw_report_line`; the preview is available, nothing is in the ledger."""
+    committed = "committed"
+    """Events written. A committed file is never re-parsed on re-upload."""
+    failed = "failed"
+    """The parser raised; `error` says what. Kept so a format regression leaves a trace."""
+
+
+class RawReportFile(TimestampMixin, Base):
+    """One uploaded file. `sha256` is the idempotency key for the whole import flow."""
+
+    __tablename__ = "raw_report_file"
+    __table_args__ = (
+        Index("uq_raw_report_file_sha256", "sha256", unique=True),
+        Index("ix_raw_report_file_account", "account_id"),
+    )
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    broker: Mapped[str] = mapped_column(String(32))
+    """sber | vtb | csv — as the parser reported it, not as the user guessed."""
+    filename: Mapped[str] = mapped_column(String(512))
+    sha256: Mapped[str] = mapped_column(String(64))
+    size_bytes: Mapped[int] = mapped_column(Integer)
+    content: Mapped[bytes | None] = mapped_column()
+    """The original bytes. Personal reports are small (tens of KB) and keeping them turns a
+    parser fix into a re-parse instead of a request to the user for a file they deleted."""
+    account_id: Mapped[int | None] = mapped_column(ForeignKey("account.id", ondelete="SET NULL"))
+    """Resolved at preview from `account_external_id`; NULL until the user picks an account."""
+    account_external_id: Mapped[str | None] = mapped_column(String(128))
+    period_from: Mapped[date | None]
+    period_to: Mapped[date | None]
+    parser_name: Mapped[str | None] = mapped_column(String(32))
+    parser_version: Mapped[str | None] = mapped_column(String(32))
+    parse_status: Mapped[ReportParseStatus] = mapped_column(
+        db_enum(ReportParseStatus, "report_parse_status"), default=ReportParseStatus.uploaded
+    )
+    error: Mapped[str | None] = mapped_column(Text)
+    warnings: Mapped[list[Any] | None]
+    counts: Mapped[dict[str, Any] | None]
+    """Preview counters kept after commit: events by kind, new/duplicate/pending totals."""
+    uploaded_at: Mapped[datetime | None]
+    committed_at: Mapped[datetime | None]
+
+
+class RawReportLine(Base):
+    """A parsed but not yet normalised row, append-only, addressable as (file, line)."""
+
+    __tablename__ = "raw_report_line"
+
+    file_id: Mapped[int] = mapped_column(
+        ForeignKey("raw_report_file.id", ondelete="CASCADE"), primary_key=True
+    )
+    line_no: Mapped[int] = mapped_column(Integer, primary_key=True)
+    section: Mapped[str | None] = mapped_column(String(64))
+    """Which part of the report the row came from — 'trades', 'cash', 'positions'."""
+    payload: Mapped[dict[str, Any]]
+    dedupe_key: Mapped[str | None] = mapped_column(String(256), index=True)
+    """The key the event will carry; kept here so the preview can spot duplicates before
+    anything is written, and so a committed line can be traced back from an event."""
+    event_id: Mapped[int | None] = mapped_column(ForeignKey("event.id", ondelete="SET NULL"))
+
+
+class PendingInstrumentStatus(enum.StrEnum):
+    pending = "pending"
+    resolved = "resolved"
+    ignored = "ignored"
+    """The user decided this key is not a tradable instrument (a fee line misread as one)."""
+
+
+class PendingInstrument(TimestampMixin, Base):
+    """An instrument a report named that nothing in the master resolves to.
+
+    Unique on (source, source_key) so the same unknown paper appearing in ten files is one
+    row with a count, not ten identical questions to the user.
+    """
+
+    __tablename__ = "pending_instrument"
+    __table_args__ = (UniqueConstraint("source", "source_key"),)
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    source: Mapped[str] = mapped_column(String(32))
+    """report_sber | report_vtb | csv"""
+    source_key: Mapped[str] = mapped_column(String(256))
+    """`InstrumentRef.key()` — 'ISIN:RU000A101EJ5', 'TICKER:STME', 'NAME:…'."""
+    isin: Mapped[str | None] = mapped_column(String(12))
+    ticker: Mapped[str | None] = mapped_column(String(32))
+    board: Mapped[str | None] = mapped_column(String(16))
+    name: Mapped[str | None] = mapped_column(String(256))
+    currency: Mapped[str | None] = mapped_column(String(3))
+    asset_class_hint: Mapped[str | None] = mapped_column(String(16))
+    """What the report's wording suggested. A hint for the UI, never applied on its own."""
+    first_seen_file_id: Mapped[int | None] = mapped_column(
+        ForeignKey("raw_report_file.id", ondelete="SET NULL")
+    )
+    occurrences: Mapped[int] = mapped_column(BigInteger, default=1)
+    sample_quantity: Mapped[Decimal | None]
+    sample_price: Mapped[Decimal | None]
+    status: Mapped[PendingInstrumentStatus] = mapped_column(
+        db_enum(PendingInstrumentStatus, "pending_instrument_status"),
+        default=PendingInstrumentStatus.pending,
+    )
+    instrument_id: Mapped[int | None] = mapped_column(
+        ForeignKey("instrument.id", ondelete="SET NULL")
+    )
+    """Set when resolved; the events waiting on this key are then bound and confirmed."""
+    resolved_at: Mapped[datetime | None]
+    meta: Mapped[dict[str, Any] | None]