"""Uploaded broker reports: the raw tier and what they could not resolve (plan §1.1, §1.3). An import is two-phase on purpose. `raw_report_file` records the bytes' sha256 the moment they arrive, so re-uploading the same file is a no-op rather than a doubled ledger — the unique index is the whole mechanism, and it works even when the file is renamed (VTB names its exports by GUID). `raw_report_line` keeps every parsed row as JSONB before normalisation, which is what makes a mapping bug diagnosable months later: the ledger can be rebuilt from these rows without asking the user for the file again. `pending_instrument` exists because guessing is worse than waiting. A report prints «Первая-ВечныйПортф БПИФ / STME» and nothing else; matching that to an instrument by name similarity would silently attach trades to the wrong paper, and a wrong lot is far more expensive to notice than an unresolved one. The row parks the unknown, the events that reference it stay `pending`, and the user confirms the instrument in the UI. """ from __future__ import annotations import enum from datetime import date, datetime from decimal import Decimal from typing import Any from sqlalchemy import BigInteger, ForeignKey, Index, Integer, String, Text, UniqueConstraint from sqlalchemy.orm import Mapped, mapped_column from fintracker.db.base import Base, TimestampMixin, db_enum class ReportParseStatus(enum.StrEnum): uploaded = "uploaded" """Bytes stored, not parsed yet.""" parsed = "parsed" """Parsed into `raw_report_line`; the preview is available, nothing is in the ledger.""" committed = "committed" """Events written. A committed file is never re-parsed on re-upload.""" failed = "failed" """The parser raised; `error` says what. Kept so a format regression leaves a trace.""" class RawReportFile(TimestampMixin, Base): """One uploaded file. `sha256` is the idempotency key for the whole import flow.""" __tablename__ = "raw_report_file" __table_args__ = ( Index("uq_raw_report_file_sha256", "sha256", unique=True), Index("ix_raw_report_file_account", "account_id"), ) id: Mapped[int] = mapped_column(primary_key=True) broker: Mapped[str] = mapped_column(String(32)) """sber | vtb | csv — as the parser reported it, not as the user guessed.""" filename: Mapped[str] = mapped_column(String(512)) sha256: Mapped[str] = mapped_column(String(64)) size_bytes: Mapped[int] = mapped_column(Integer) content: Mapped[bytes | None] = mapped_column() """The original bytes. Personal reports are small (tens of KB) and keeping them turns a parser fix into a re-parse instead of a request to the user for a file they deleted.""" account_id: Mapped[int | None] = mapped_column(ForeignKey("account.id", ondelete="SET NULL")) """Resolved at preview from `account_external_id`; NULL until the user picks an account.""" account_external_id: Mapped[str | None] = mapped_column(String(128)) period_from: Mapped[date | None] period_to: Mapped[date | None] parser_name: Mapped[str | None] = mapped_column(String(32)) parser_version: Mapped[str | None] = mapped_column(String(32)) parse_status: Mapped[ReportParseStatus] = mapped_column( db_enum(ReportParseStatus, "report_parse_status"), default=ReportParseStatus.uploaded ) error: Mapped[str | None] = mapped_column(Text) warnings: Mapped[list[Any] | None] counts: Mapped[dict[str, Any] | None] """Preview counters kept after commit: events by kind, new/duplicate/pending totals.""" uploaded_at: Mapped[datetime | None] committed_at: Mapped[datetime | None] class RawReportLine(Base): """A parsed but not yet normalised row, append-only, addressable as (file, line).""" __tablename__ = "raw_report_line" file_id: Mapped[int] = mapped_column( ForeignKey("raw_report_file.id", ondelete="CASCADE"), primary_key=True ) line_no: Mapped[int] = mapped_column(Integer, primary_key=True) section: Mapped[str | None] = mapped_column(String(64)) """Which part of the report the row came from — 'trades', 'cash', 'positions'.""" payload: Mapped[dict[str, Any]] dedupe_key: Mapped[str | None] = mapped_column(String(256), index=True) """The key the event will carry; kept here so the preview can spot duplicates before anything is written, and so a committed line can be traced back from an event.""" event_id: Mapped[int | None] = mapped_column(ForeignKey("event.id", ondelete="SET NULL")) class PendingInstrumentStatus(enum.StrEnum): pending = "pending" resolved = "resolved" ignored = "ignored" """The user decided this key is not a tradable instrument (a fee line misread as one).""" class PendingInstrument(TimestampMixin, Base): """An instrument a report named that nothing in the master resolves to. Unique on (source, source_key) so the same unknown paper appearing in ten files is one row with a count, not ten identical questions to the user. """ __tablename__ = "pending_instrument" __table_args__ = (UniqueConstraint("source", "source_key"),) id: Mapped[int] = mapped_column(primary_key=True) source: Mapped[str] = mapped_column(String(32)) """report_sber | report_vtb | csv""" source_key: Mapped[str] = mapped_column(String(256)) """`InstrumentRef.key()` — 'ISIN:RU000A101EJ5', 'TICKER:STME', 'NAME:…'.""" isin: Mapped[str | None] = mapped_column(String(12)) ticker: Mapped[str | None] = mapped_column(String(32)) board: Mapped[str | None] = mapped_column(String(16)) name: Mapped[str | None] = mapped_column(String(256)) currency: Mapped[str | None] = mapped_column(String(3)) asset_class_hint: Mapped[str | None] = mapped_column(String(16)) """What the report's wording suggested. A hint for the UI, never applied on its own.""" first_seen_file_id: Mapped[int | None] = mapped_column( ForeignKey("raw_report_file.id", ondelete="SET NULL") ) occurrences: Mapped[int] = mapped_column(BigInteger, default=1) sample_quantity: Mapped[Decimal | None] sample_price: Mapped[Decimal | None] status: Mapped[PendingInstrumentStatus] = mapped_column( db_enum(PendingInstrumentStatus, "pending_instrument_status"), default=PendingInstrumentStatus.pending, ) instrument_id: Mapped[int | None] = mapped_column( ForeignKey("instrument.id", ondelete="SET NULL") ) """Set when resolved; the events waiting on this key are then bound and confirmed.""" resolved_at: Mapped[datetime | None] meta: Mapped[dict[str, Any] | None]