fin-tracker/backend/src/fintracker/models/reports.py

"""Uploaded broker reports: the raw tier and what they could not resolve (plan §1.1, §1.3).

An import is two-phase on purpose. `raw_report_file` records the bytes' sha256 the moment
they arrive, so re-uploading the same file is a no-op rather than a doubled ledger — the
unique index is the whole mechanism, and it works even when the file is renamed (VTB names
its exports by GUID). `raw_report_line` keeps every parsed row as JSONB before normalisation,
which is what makes a mapping bug diagnosable months later: the ledger can be rebuilt from
these rows without asking the user for the file again.

`pending_instrument` exists because guessing is worse than waiting. A report prints
«Первая-ВечныйПортф БПИФ / STME» and nothing else; matching that to an instrument by name
similarity would silently attach trades to the wrong paper, and a wrong lot is far more
expensive to notice than an unresolved one. The row parks the unknown, the events that
reference it stay `pending`, and the user confirms the instrument in the UI.
"""

from __future__ import annotations

import enum
from datetime import date, datetime
from decimal import Decimal
from typing import Any

from sqlalchemy import BigInteger, ForeignKey, Index, Integer, String, Text, UniqueConstraint
from sqlalchemy.orm import Mapped, mapped_column

from fintracker.db.base import Base, TimestampMixin, db_enum


class ReportParseStatus(enum.StrEnum):
    uploaded = "uploaded"
    """Bytes stored, not parsed yet."""
    parsed = "parsed"
    """Parsed into `raw_report_line`; the preview is available, nothing is in the ledger."""
    committed = "committed"
    """Events written. A committed file is never re-parsed on re-upload."""
    failed = "failed"
    """The parser raised; `error` says what. Kept so a format regression leaves a trace."""


class RawReportFile(TimestampMixin, Base):
    """One uploaded file. `sha256` is the idempotency key for the whole import flow."""

    __tablename__ = "raw_report_file"
    __table_args__ = (
        Index("uq_raw_report_file_sha256", "sha256", unique=True),
        Index("ix_raw_report_file_account", "account_id"),
    )

    id: Mapped[int] = mapped_column(primary_key=True)
    broker: Mapped[str] = mapped_column(String(32))
    """sber | vtb | csv — as the parser reported it, not as the user guessed."""
    filename: Mapped[str] = mapped_column(String(512))
    sha256: Mapped[str] = mapped_column(String(64))
    size_bytes: Mapped[int] = mapped_column(Integer)
    content: Mapped[bytes | None] = mapped_column()
    """The original bytes. Personal reports are small (tens of KB) and keeping them turns a
    parser fix into a re-parse instead of a request to the user for a file they deleted."""
    account_id: Mapped[int | None] = mapped_column(ForeignKey("account.id", ondelete="SET NULL"))
    """Resolved at preview from `account_external_id`; NULL until the user picks an account."""
    account_external_id: Mapped[str | None] = mapped_column(String(128))
    period_from: Mapped[date | None]
    period_to: Mapped[date | None]
    parser_name: Mapped[str | None] = mapped_column(String(32))
    parser_version: Mapped[str | None] = mapped_column(String(32))
    parse_status: Mapped[ReportParseStatus] = mapped_column(
        db_enum(ReportParseStatus, "report_parse_status"), default=ReportParseStatus.uploaded
    )
    error: Mapped[str | None] = mapped_column(Text)
    warnings: Mapped[list[Any] | None]
    counts: Mapped[dict[str, Any] | None]
    """Preview counters kept after commit: events by kind, new/duplicate/pending totals."""
    uploaded_at: Mapped[datetime | None]
    committed_at: Mapped[datetime | None]


class RawReportLine(Base):
    """A parsed but not yet normalised row, append-only, addressable as (file, line)."""

    __tablename__ = "raw_report_line"

    file_id: Mapped[int] = mapped_column(
        ForeignKey("raw_report_file.id", ondelete="CASCADE"), primary_key=True
    )
    line_no: Mapped[int] = mapped_column(Integer, primary_key=True)
    section: Mapped[str | None] = mapped_column(String(64))
    """Which part of the report the row came from — 'trades', 'cash', 'positions'."""
    payload: Mapped[dict[str, Any]]
    dedupe_key: Mapped[str | None] = mapped_column(String(256), index=True)
    """The key the event will carry; kept here so the preview can spot duplicates before
    anything is written, and so a committed line can be traced back from an event."""
    event_id: Mapped[int | None] = mapped_column(ForeignKey("event.id", ondelete="SET NULL"))


class PendingInstrumentStatus(enum.StrEnum):
    pending = "pending"
    resolved = "resolved"
    ignored = "ignored"
    """The user decided this key is not a tradable instrument (a fee line misread as one)."""


class PendingInstrument(TimestampMixin, Base):
    """An instrument a report named that nothing in the master resolves to.

    Unique on (source, source_key) so the same unknown paper appearing in ten files is one
    row with a count, not ten identical questions to the user.
    """

    __tablename__ = "pending_instrument"
    __table_args__ = (UniqueConstraint("source", "source_key"),)

    id: Mapped[int] = mapped_column(primary_key=True)
    source: Mapped[str] = mapped_column(String(32))
    """report_sber | report_vtb | csv"""
    source_key: Mapped[str] = mapped_column(String(256))
    """`InstrumentRef.key()` — 'ISIN:RU000A101EJ5', 'TICKER:STME', 'NAME:…'."""
    isin: Mapped[str | None] = mapped_column(String(12))
    ticker: Mapped[str | None] = mapped_column(String(32))
    board: Mapped[str | None] = mapped_column(String(16))
    name: Mapped[str | None] = mapped_column(String(256))
    currency: Mapped[str | None] = mapped_column(String(3))
    asset_class_hint: Mapped[str | None] = mapped_column(String(16))
    """What the report's wording suggested. A hint for the UI, never applied on its own."""
    first_seen_file_id: Mapped[int | None] = mapped_column(
        ForeignKey("raw_report_file.id", ondelete="SET NULL")
    )
    occurrences: Mapped[int] = mapped_column(BigInteger, default=1)
    sample_quantity: Mapped[Decimal | None]
    sample_price: Mapped[Decimal | None]
    status: Mapped[PendingInstrumentStatus] = mapped_column(
        db_enum(PendingInstrumentStatus, "pending_instrument_status"),
        default=PendingInstrumentStatus.pending,
    )
    instrument_id: Mapped[int | None] = mapped_column(
        ForeignKey("instrument.id", ondelete="SET NULL")
    )
    """Set when resolved; the events waiting on this key are then bound and confirmed."""
    resolved_at: Mapped[datetime | None]
    meta: Mapped[dict[str, Any] | None]