fin-tracker/backend/tests/ledger/test_ingest.py

"""`BrokerEvent` -> `event`: idempotency, pending instruments, shadow status, side effects.

Every `ParsedReport` here is built by hand: the parsers are a separate contract and a bug in
one of them must not be able to fail these tests, which are about what the ledger does with
what a parser produced.
"""

from __future__ import annotations

from datetime import date
from decimal import Decimal

import pytest
from sqlalchemy import func, select

from factories import make_instrument
from fintracker.db import get_sessionmaker
from fintracker.ledger.ingest import ingest
from fintracker.ledger.report_import import InstrumentSpec, resolve_pending
from fintracker.models import (
    Account,
    AccountKind,
    AccountRole,
    AssetClass,
    Event,
    EventKind,
    EventSource,
    EventStatus,
    Instrument,
    InstrumentAlias,
    Lot,
    PendingInstrument,
    PendingInstrumentStatus,
    PriceManual,
    RawReportFile,
    RawReportLine,
    ReportParseStatus,
)
from fintracker.sources.reports.base import (
    BrokerEvent,
    InstrumentRef,
    ParsedReport,
)

SOURCE = "report_sber"
ACCOUNT_NO = "S930W42"
UNKNOWN_ISIN = "RU000A1035S8"


async def make_broker_account(
    *, primary: EventSource | None = EventSource.report_sber, source: str = SOURCE
) -> int:
    async with get_sessionmaker()() as session:
        account = Account(
            kind=AccountKind.broker,
            source=source,
            source_id=ACCOUNT_NO,
            name="Сбер ИИС",
            currency="RUB",
            role=AccountRole.investment,
            include_in_net_worth=False,
            primary_event_source=primary,
        )
        session.add(account)
        await session.commit()
        return account.id


async def make_file(account_id: int | None = None) -> int:
    async with get_sessionmaker()() as session:
        row = RawReportFile(
            broker="sber",
            filename="report.html",
            sha256="0" * 64,
            size_bytes=10,
            parser_name=SOURCE,
            parser_version="1",
            parse_status=ReportParseStatus.parsed,
            account_id=account_id,
        )
        session.add(row)
        await session.commit()
        return row.id


def unknown_ref() -> InstrumentRef:
    return InstrumentRef(
        isin=UNKNOWN_ISIN,
        ticker="STME",
        name="Первая-ВечныйПортф БПИФ",
        currency="RUB",
        asset_class_hint="fund",
        source_key=f"ISIN:{UNKNOWN_ISIN}",
    )


def report(
    events: list[BrokerEvent],
    *,
    meta: dict | None = None,
    instruments: list[InstrumentRef] | None = None,
) -> ParsedReport:
    return ParsedReport(
        broker="sber",
        account_external_id=ACCOUNT_NO,
        period_from=date(2026, 2, 11),
        period_to=date(2026, 9, 17),
        parser_version="1",
        events=events,
        instruments=instruments or [],
        meta=meta or {},
    )


def buy(
    ref: InstrumentRef | None,
    *,
    trade_no: str | None = "15678045077",
    d: date = date(2026, 2, 24),
    qty: str = "10",
    price: str = "100",
    line_no: int = 1,
) -> BrokerEvent:
    return BrokerEvent(
        kind=EventKind.buy,
        trade_date=d,
        settle_date=d,
        amount=Decimal(qty) * Decimal(price) * -1,
        currency="RUB",
        instrument=ref,
        quantity=Decimal(qty),
        price=Decimal(price),
        price_currency="RUB",
        fee=Decimal("0.39"),
        trade_no=trade_no,
        description="Покупка",
        raw_line_no=line_no,
    )


async def run_ingest(account_id: int, parsed: ParsedReport, *, file_id: int | None = None):
    async with get_sessionmaker()() as session:
        account = await session.get(Account, account_id)
        assert account is not None
        result = await ingest(
            session, account=account, parsed=parsed, source=SOURCE, file_id=file_id
        )
        await session.commit()
        return result


async def count(model, *where) -> int:
    async with get_sessionmaker()() as session:
        return await session.scalar(select(func.count()).select_from(model).where(*where)) or 0


async def test_same_report_twice_creates_nothing_the_second_time(app):
    account_id = await make_broker_account()
    instrument_id = await make_instrument(ticker="GAZP", name="Газпром")
    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
    parsed = report([buy(ref)])

    first = await run_ingest(account_id, parsed)
    second = await run_ingest(account_id, parsed)

    assert first.events_created == 1
    assert second.events_created == 0
    assert second.events_updated == 1
    assert second.events_duplicate == 1
    assert await count(Event) == 1
    async with get_sessionmaker()() as session:
        event = (await session.execute(select(Event))).scalar_one()
    assert event.instrument_id == instrument_id
    assert event.status == EventStatus.confirmed


async def test_unknown_isin_parks_the_instrument_and_counts_repeats(app):
    account_id = await make_broker_account()
    parsed = report([buy(unknown_ref())])

    first = await run_ingest(account_id, parsed)
    assert first.events_pending == 1

    async with get_sessionmaker()() as session:
        event = (await session.execute(select(Event))).scalar_one()
        pending = (await session.execute(select(PendingInstrument))).scalar_one()
    assert event.status == EventStatus.pending
    assert event.instrument_id is None
    assert (event.meta or {})["pending_key"] == f"ISIN:{UNKNOWN_ISIN}"
    assert pending.source == SOURCE
    assert pending.isin == UNKNOWN_ISIN
    assert pending.occurrences == 1

    await run_ingest(account_id, parsed)
    async with get_sessionmaker()() as session:
        rows = (await session.execute(select(PendingInstrument))).scalars().all()
    assert len(rows) == 1
    assert rows[0].occurrences == 2


async def test_resolving_a_pending_instrument_binds_and_confirms_its_events(app):
    account_id = await make_broker_account()
    await run_ingest(account_id, report([buy(unknown_ref())]))
    instrument_id = await make_instrument(ticker="STME", name="БПИФ", board="TQTF")

    async with get_sessionmaker()() as session:
        pending = (await session.execute(select(PendingInstrument))).scalar_one()
        outcome = await resolve_pending(
            session, pending, action="link", instrument_id=instrument_id
        )
    assert outcome.events_bound == 1
    assert outcome.alias_created is True

    async with get_sessionmaker()() as session:
        event = (await session.execute(select(Event))).scalar_one()
        alias = (await session.execute(select(InstrumentAlias))).scalar_one()
        lots = (await session.execute(select(Lot))).scalars().all()
        pending = (await session.execute(select(PendingInstrument))).scalar_one()
    assert event.status == EventStatus.confirmed
    assert event.instrument_id == instrument_id
    assert "pending_key" not in (event.meta or {})
    assert (alias.source, alias.source_key) == (SOURCE, f"ISIN:{UNKNOWN_ISIN}")
    assert pending.status == PendingInstrumentStatus.resolved
    assert [(lot.instrument_id, lot.qty_open) for lot in lots] == [(instrument_id, Decimal(10))]


async def test_report_on_an_api_primary_account_lands_as_shadow(app):
    account_id = await make_broker_account(primary=EventSource.tinvest_api, source="tinvest")
    await make_instrument(ticker="GAZP", name="Газпром")
    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")

    result = await run_ingest(account_id, report([buy(ref)]))

    assert result.events_shadow == 1
    async with get_sessionmaker()() as session:
        event = (await session.execute(select(Event))).scalar_one()
    assert event.status == EventStatus.shadow


async def test_manual_prices_from_meta_become_price_manual_rows(app):
    account_id = await make_broker_account()
    instrument_id = await make_instrument(ticker="GAZP", name="Газпром")
    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
    parsed = report(
        [buy(ref)],
        meta={
            "manual_prices": [
                {
                    "instrument_key": "TICKER:GAZP/TQBR",
                    "d": "2026-09-17",
                    "price": "123.45",
                    "currency": "RUB",
                }
            ]
        },
    )

    result = await run_ingest(account_id, parsed)

    assert result.manual_prices == 1
    async with get_sessionmaker()() as session:
        row = (await session.execute(select(PriceManual))).scalar_one()
    assert row.instrument_id == instrument_id
    assert row.d == date(2026, 9, 17)
    assert row.price == Decimal("123.45")


async def test_payout_hint_turns_a_dividend_on_a_bond_into_a_coupon(app):
    account_id = await make_broker_account()
    bond_id = await make_instrument(
        ticker="SU26238", name="ОФЗ 26238", asset_class=AssetClass.bond, board="TQOB"
    )
    ref = InstrumentRef(ticker="SU26238", board="TQOB", source_key="TICKER:SU26238/TQOB")
    payout = BrokerEvent(
        kind=EventKind.dividend,
        trade_date=date(2026, 5, 20),
        amount=Decimal("175.30"),
        currency="RUB",
        instrument=ref,
        trade_no="PAY-1",
        meta={"payout_hint": "coupon"},
        raw_line_no=7,
    )

    await run_ingest(account_id, report([payout]))

    async with get_sessionmaker()() as session:
        event = (await session.execute(select(Event))).scalar_one()
    assert event.kind == EventKind.coupon
    assert event.instrument_id == bond_id


async def test_raw_report_lines_are_linked_and_carry_no_float(app):
    account_id = await make_broker_account()
    await make_instrument(ticker="GAZP", name="Газпром")
    ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
    file_id = await make_file(account_id)

    await run_ingest(account_id, report([buy(ref, line_no=3)]), file_id=file_id)

    async with get_sessionmaker()() as session:
        line = (await session.execute(select(RawReportLine))).scalar_one()
        event = (await session.execute(select(Event))).scalar_one()
    assert line.file_id == file_id
    assert line.line_no == 3
    assert line.event_id == event.id
    assert line.dedupe_key == event.dedupe_key
    assert line.payload["quantity"] == "10"
    assert _floats(line.payload) == []


def _floats(value, path: str = "$") -> list[str]:
    """Every float hiding in a JSONB payload, by path — money must never be one."""
    if isinstance(value, float):
        return [path]
    if isinstance(value, dict):
        return [p for k, v in value.items() for p in _floats(v, f"{path}.{k}")]
    if isinstance(value, list):
        return [p for i, v in enumerate(value) for p in _floats(v, f"{path}[{i}]")]
    return []


@pytest.mark.parametrize(
    ("spec", "expected"),
    [(InstrumentSpec(asset_class="fund", name="X"), AssetClass.fund)],
)
async def test_create_mode_builds_the_instrument_from_the_report(app, spec, expected):
    account_id = await make_broker_account()
    await run_ingest(account_id, report([buy(unknown_ref())]))

    async with get_sessionmaker()() as session:
        pending = (await session.execute(select(PendingInstrument))).scalar_one()
        outcome = await resolve_pending(session, pending, action="create", instrument=spec)

    async with get_sessionmaker()() as session:
        instrument = await session.get(Instrument, outcome.instrument_id)
    assert instrument is not None
    assert instrument.asset_class == expected