feat(ledger): импорт отчётов в леджер — приём, дедупликация, pending_instrument

Поток: upload -> raw_report_file (sha256 UNIQUE) -> parse -> raw_report_line, событий в леджере ещё нет -> preview -> POST /imports/{id}/commit -> ledger/ingest.py резолвит инструмент, считает dedupe_key, пишет event confirmed | shadow (plan §1.6 B) — сравнивая account.primary_event_source с источником отчёта, а не гадая. Нерезолвленный инструмент ждёт в pending_instrument, никогда не угадывается; POST /instruments/pending/{id}/resolve привязывает и пересобирает лоты. ledger/dedupe.py — shadow-матчинг случая B двумя проходами (точная дата, затем ±1 рабочий день, жадно 1:1, |price| ±0,5 %). Шаги shadow_dedupe и report_reconcile зарегистрированы перед quality: оба говорят через FINDINGS. /instruments/pending регистрируется в app.py ДО routers/instruments.py: FastAPI сопоставляет маршруты по порядку, и /instruments/{id} с типом int отвечает 422 на нечисловой сегмент, а не проваливается дальше. Контракт — docs/ai/import-contract.md, общий для бэкенда и Flutter.
2026-09-19 10:40:04 +03:00
parent 2e742a093b
commit ff3b76871d
16 changed files with 4484 additions and 0 deletions
@@ -0,0 +1,338 @@
+"""`BrokerEvent` -> `event`: idempotency, pending instruments, shadow status, side effects.
+
+Every `ParsedReport` here is built by hand: the parsers are a separate contract and a bug in
+one of them must not be able to fail these tests, which are about what the ledger does with
+what a parser produced.
+"""
+
+from __future__ import annotations
+
+from datetime import date
+from decimal import Decimal
+
+import pytest
+from sqlalchemy import func, select
+
+from factories import make_instrument
+from fintracker.db import get_sessionmaker
+from fintracker.ledger.ingest import ingest
+from fintracker.ledger.report_import import InstrumentSpec, resolve_pending
+from fintracker.models import (
+    Account,
+    AccountKind,
+    AccountRole,
+    AssetClass,
+    Event,
+    EventKind,
+    EventSource,
+    EventStatus,
+    Instrument,
+    InstrumentAlias,
+    Lot,
+    PendingInstrument,
+    PendingInstrumentStatus,
+    PriceManual,
+    RawReportFile,
+    RawReportLine,
+    ReportParseStatus,
+)
+from fintracker.sources.reports.base import (
+    BrokerEvent,
+    InstrumentRef,
+    ParsedReport,
+)
+
+SOURCE = "report_sber"
+ACCOUNT_NO = "S930W42"
+UNKNOWN_ISIN = "RU000A1035S8"
+
+
+async def make_broker_account(
+    *, primary: EventSource | None = EventSource.report_sber, source: str = SOURCE
+) -> int:
+    async with get_sessionmaker()() as session:
+        account = Account(
+            kind=AccountKind.broker,
+            source=source,
+            source_id=ACCOUNT_NO,
+            name="Сбер ИИС",
+            currency="RUB",
+            role=AccountRole.investment,
+            include_in_net_worth=False,
+            primary_event_source=primary,
+        )
+        session.add(account)
+        await session.commit()
+        return account.id
+
+
+async def make_file(account_id: int | None = None) -> int:
+    async with get_sessionmaker()() as session:
+        row = RawReportFile(
+            broker="sber",
+            filename="report.html",
+            sha256="0" * 64,
+            size_bytes=10,
+            parser_name=SOURCE,
+            parser_version="1",
+            parse_status=ReportParseStatus.parsed,
+            account_id=account_id,
+        )
+        session.add(row)
+        await session.commit()
+        return row.id
+
+
+def unknown_ref() -> InstrumentRef:
+    return InstrumentRef(
+        isin=UNKNOWN_ISIN,
+        ticker="STME",
+        name="Первая-ВечныйПортф БПИФ",
+        currency="RUB",
+        asset_class_hint="fund",
+        source_key=f"ISIN:{UNKNOWN_ISIN}",
+    )
+
+
+def report(
+    events: list[BrokerEvent],
+    *,
+    meta: dict | None = None,
+    instruments: list[InstrumentRef] | None = None,
+) -> ParsedReport:
+    return ParsedReport(
+        broker="sber",
+        account_external_id=ACCOUNT_NO,
+        period_from=date(2026, 2, 11),
+        period_to=date(2026, 9, 17),
+        parser_version="1",
+        events=events,
+        instruments=instruments or [],
+        meta=meta or {},
+    )
+
+
+def buy(
+    ref: InstrumentRef | None,
+    *,
+    trade_no: str | None = "15678045077",
+    d: date = date(2026, 2, 24),
+    qty: str = "10",
+    price: str = "100",
+    line_no: int = 1,
+) -> BrokerEvent:
+    return BrokerEvent(
+        kind=EventKind.buy,
+        trade_date=d,
+        settle_date=d,
+        amount=Decimal(qty) * Decimal(price) * -1,
+        currency="RUB",
+        instrument=ref,
+        quantity=Decimal(qty),
+        price=Decimal(price),
+        price_currency="RUB",
+        fee=Decimal("0.39"),
+        trade_no=trade_no,
+        description="Покупка",
+        raw_line_no=line_no,
+    )
+
+
+async def run_ingest(account_id: int, parsed: ParsedReport, *, file_id: int | None = None):
+    async with get_sessionmaker()() as session:
+        account = await session.get(Account, account_id)
+        assert account is not None
+        result = await ingest(
+            session, account=account, parsed=parsed, source=SOURCE, file_id=file_id
+        )
+        await session.commit()
+        return result
+
+
+async def count(model, *where) -> int:
+    async with get_sessionmaker()() as session:
+        return await session.scalar(select(func.count()).select_from(model).where(*where)) or 0
+
+
+async def test_same_report_twice_creates_nothing_the_second_time(app):
+    account_id = await make_broker_account()
+    instrument_id = await make_instrument(ticker="GAZP", name="Газпром")
+    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
+    parsed = report([buy(ref)])
+
+    first = await run_ingest(account_id, parsed)
+    second = await run_ingest(account_id, parsed)
+
+    assert first.events_created == 1
+    assert second.events_created == 0
+    assert second.events_updated == 1
+    assert second.events_duplicate == 1
+    assert await count(Event) == 1
+    async with get_sessionmaker()() as session:
+        event = (await session.execute(select(Event))).scalar_one()
+    assert event.instrument_id == instrument_id
+    assert event.status == EventStatus.confirmed
+
+
+async def test_unknown_isin_parks_the_instrument_and_counts_repeats(app):
+    account_id = await make_broker_account()
+    parsed = report([buy(unknown_ref())])
+
+    first = await run_ingest(account_id, parsed)
+    assert first.events_pending == 1
+
+    async with get_sessionmaker()() as session:
+        event = (await session.execute(select(Event))).scalar_one()
+        pending = (await session.execute(select(PendingInstrument))).scalar_one()
+    assert event.status == EventStatus.pending
+    assert event.instrument_id is None
+    assert (event.meta or {})["pending_key"] == f"ISIN:{UNKNOWN_ISIN}"
+    assert pending.source == SOURCE
+    assert pending.isin == UNKNOWN_ISIN
+    assert pending.occurrences == 1
+
+    await run_ingest(account_id, parsed)
+    async with get_sessionmaker()() as session:
+        rows = (await session.execute(select(PendingInstrument))).scalars().all()
+    assert len(rows) == 1
+    assert rows[0].occurrences == 2
+
+
+async def test_resolving_a_pending_instrument_binds_and_confirms_its_events(app):
+    account_id = await make_broker_account()
+    await run_ingest(account_id, report([buy(unknown_ref())]))
+    instrument_id = await make_instrument(ticker="STME", name="БПИФ", board="TQTF")
+
+    async with get_sessionmaker()() as session:
+        pending = (await session.execute(select(PendingInstrument))).scalar_one()
+        outcome = await resolve_pending(
+            session, pending, action="link", instrument_id=instrument_id
+        )
+    assert outcome.events_bound == 1
+    assert outcome.alias_created is True
+
+    async with get_sessionmaker()() as session:
+        event = (await session.execute(select(Event))).scalar_one()
+        alias = (await session.execute(select(InstrumentAlias))).scalar_one()
+        lots = (await session.execute(select(Lot))).scalars().all()
+        pending = (await session.execute(select(PendingInstrument))).scalar_one()
+    assert event.status == EventStatus.confirmed
+    assert event.instrument_id == instrument_id
+    assert "pending_key" not in (event.meta or {})
+    assert (alias.source, alias.source_key) == (SOURCE, f"ISIN:{UNKNOWN_ISIN}")
+    assert pending.status == PendingInstrumentStatus.resolved
+    assert [(lot.instrument_id, lot.qty_open) for lot in lots] == [(instrument_id, Decimal(10))]
+
+
+async def test_report_on_an_api_primary_account_lands_as_shadow(app):
+    account_id = await make_broker_account(primary=EventSource.tinvest_api, source="tinvest")
+    await make_instrument(ticker="GAZP", name="Газпром")
+    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
+
+    result = await run_ingest(account_id, report([buy(ref)]))
+
+    assert result.events_shadow == 1
+    async with get_sessionmaker()() as session:
+        event = (await session.execute(select(Event))).scalar_one()
+    assert event.status == EventStatus.shadow
+
+
+async def test_manual_prices_from_meta_become_price_manual_rows(app):
+    account_id = await make_broker_account()
+    instrument_id = await make_instrument(ticker="GAZP", name="Газпром")
+    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
+    parsed = report(
+        [buy(ref)],
+        meta={
+            "manual_prices": [
+                {
+                    "instrument_key": "TICKER:GAZP/TQBR",
+                    "d": "2026-09-17",
+                    "price": "123.45",
+                    "currency": "RUB",
+                }
+            ]
+        },
+    )
+
+    result = await run_ingest(account_id, parsed)
+
+    assert result.manual_prices == 1
+    async with get_sessionmaker()() as session:
+        row = (await session.execute(select(PriceManual))).scalar_one()
+    assert row.instrument_id == instrument_id
+    assert row.d == date(2026, 9, 17)
+    assert row.price == Decimal("123.45")
+
+
+async def test_payout_hint_turns_a_dividend_on_a_bond_into_a_coupon(app):
+    account_id = await make_broker_account()
+    bond_id = await make_instrument(
+        ticker="SU26238", name="ОФЗ 26238", asset_class=AssetClass.bond, board="TQOB"
+    )
+    ref = InstrumentRef(ticker="SU26238", board="TQOB", source_key="TICKER:SU26238/TQOB")
+    payout = BrokerEvent(
+        kind=EventKind.dividend,
+        trade_date=date(2026, 5, 20),
+        amount=Decimal("175.30"),
+        currency="RUB",
+        instrument=ref,
+        trade_no="PAY-1",
+        meta={"payout_hint": "coupon"},
+        raw_line_no=7,
+    )
+
+    await run_ingest(account_id, report([payout]))
+
+    async with get_sessionmaker()() as session:
+        event = (await session.execute(select(Event))).scalar_one()
+    assert event.kind == EventKind.coupon
+    assert event.instrument_id == bond_id
+
+
+async def test_raw_report_lines_are_linked_and_carry_no_float(app):
+    account_id = await make_broker_account()
+    await make_instrument(ticker="GAZP", name="Газпром")
+    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
+    file_id = await make_file(account_id)
+
+    await run_ingest(account_id, report([buy(ref, line_no=3)]), file_id=file_id)
+
+    async with get_sessionmaker()() as session:
+        line = (await session.execute(select(RawReportLine))).scalar_one()
+        event = (await session.execute(select(Event))).scalar_one()
+    assert line.file_id == file_id
+    assert line.line_no == 3
+    assert line.event_id == event.id
+    assert line.dedupe_key == event.dedupe_key
+    assert line.payload["quantity"] == "10"
+    assert _floats(line.payload) == []
+
+
+def _floats(value, path: str = "$") -> list[str]:
+    """Every float hiding in a JSONB payload, by path — money must never be one."""
+    if isinstance(value, float):
+        return [path]
+    if isinstance(value, dict):
+        return [p for k, v in value.items() for p in _floats(v, f"{path}.{k}")]
+    if isinstance(value, list):
+        return [p for i, v in enumerate(value) for p in _floats(v, f"{path}[{i}]")]
+    return []
+
+
+@pytest.mark.parametrize(
+    ("spec", "expected"),
+    [(InstrumentSpec(asset_class="fund", name="X"), AssetClass.fund)],
+)
+async def test_create_mode_builds_the_instrument_from_the_report(app, spec, expected):
+    account_id = await make_broker_account()
+    await run_ingest(account_id, report([buy(unknown_ref())]))
+
+    async with get_sessionmaker()() as session:
+        pending = (await session.execute(select(PendingInstrument))).scalar_one()
+        outcome = await resolve_pending(session, pending, action="create", instrument=spec)
+
+    async with get_sessionmaker()() as session:
+        instrument = await session.get(Instrument, outcome.instrument_id)
+    assert instrument is not None
+    assert instrument.asset_class == expected