feat(ledger): импорт отчётов в леджер — приём, дедупликация, pending_instrument
Поток: upload -> raw_report_file (sha256 UNIQUE) -> parse -> raw_report_line,
событий в леджере ещё нет -> preview -> POST /imports/{id}/commit ->
ledger/ingest.py резолвит инструмент, считает dedupe_key, пишет event
confirmed | shadow (plan §1.6 B) — сравнивая account.primary_event_source
с источником отчёта, а не гадая. Нерезолвленный инструмент ждёт в
pending_instrument, никогда не угадывается; POST /instruments/pending/{id}/resolve
привязывает и пересобирает лоты.
ledger/dedupe.py — shadow-матчинг случая B двумя проходами (точная дата, затем
±1 рабочий день, жадно 1:1, |price| ±0,5 %). Шаги shadow_dedupe и
report_reconcile зарегистрированы перед quality: оба говорят через FINDINGS.
/instruments/pending регистрируется в app.py ДО routers/instruments.py:
FastAPI сопоставляет маршруты по порядку, и /instruments/{id} с типом int
отвечает 422 на нечисловой сегмент, а не проваливается дальше.
Контракт — docs/ai/import-contract.md, общий для бэкенда и Flutter.
This commit is contained in:
Dmitry
@@ -0,0 +1,241 @@
|
||||
"""Shadow matching: which report rows are the API's rows seen twice, and which are news."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from factories import make_account, make_event, make_instrument
|
||||
from fintracker.analytics import FINDINGS
|
||||
from fintracker.db import get_sessionmaker
|
||||
from fintracker.ledger.dedupe import match_shadow_events
|
||||
from fintracker.models import AccountKind, AccountRole, Event, EventKind, EventStatus
|
||||
|
||||
TRADE_DAY = date(2026, 3, 10)
|
||||
|
||||
|
||||
async def broker_account() -> int:
|
||||
return await make_account(
|
||||
name="Брокерский",
|
||||
kind=AccountKind.broker,
|
||||
role=AccountRole.investment,
|
||||
balance=None,
|
||||
include_in_net_worth=False,
|
||||
source="tinvest",
|
||||
)
|
||||
|
||||
|
||||
async def run() -> tuple[int, int]:
|
||||
FINDINGS.reset()
|
||||
async with get_sessionmaker()() as session:
|
||||
result = await match_shadow_events(session)
|
||||
await session.commit()
|
||||
return result.matched, result.unmatched
|
||||
|
||||
|
||||
async def matched_ids() -> dict[int, int | None]:
|
||||
async with get_sessionmaker()() as session:
|
||||
rows = (
|
||||
await session.execute(select(Event).where(Event.status == EventStatus.shadow))
|
||||
).scalars()
|
||||
return {e.id: (e.meta or {}).get("matched_event_id") for e in rows}
|
||||
|
||||
|
||||
async def test_price_within_half_a_percent_is_the_same_trade(app):
|
||||
account = await broker_account()
|
||||
instrument = await make_instrument(ticker="GAZP")
|
||||
api = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
shadow = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100.4",
|
||||
amount="-1004",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
|
||||
matched, unmatched = await run()
|
||||
|
||||
assert (matched, unmatched) == (1, 0)
|
||||
assert await matched_ids() == {shadow: api}
|
||||
assert FINDINGS.items == []
|
||||
|
||||
|
||||
async def test_price_off_by_two_percent_is_not_matched_and_is_reported(app):
|
||||
account = await broker_account()
|
||||
instrument = await make_instrument(ticker="GAZP")
|
||||
await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
shadow = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="102",
|
||||
amount="-1020",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
|
||||
matched, unmatched = await run()
|
||||
|
||||
assert (matched, unmatched) == (0, 1)
|
||||
assert await matched_ids() == {shadow: None}
|
||||
assert [f.check_name for f in FINDINGS.items] == ["report_only_event"]
|
||||
assert "нет в API" in FINDINGS.items[0].detail
|
||||
|
||||
|
||||
async def test_one_confirmed_event_closes_only_one_shadow(app):
|
||||
account = await broker_account()
|
||||
instrument = await make_instrument(ticker="GAZP")
|
||||
await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
first = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
second = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
|
||||
matched, unmatched = await run()
|
||||
|
||||
assert (matched, unmatched) == (1, 1)
|
||||
marks = await matched_ids()
|
||||
assert sorted(marks) == sorted([first, second])
|
||||
assert sum(1 for v in marks.values() if v is not None) == 1
|
||||
assert [f.check_name for f in FINDINGS.items] == ["report_only_event"]
|
||||
|
||||
|
||||
async def test_settlement_date_one_business_day_later_still_matches(app):
|
||||
"""The report prints T+1 where the API prints T — the relaxed level exists for this."""
|
||||
account = await broker_account()
|
||||
instrument = await make_instrument(ticker="GAZP")
|
||||
api = await make_event(
|
||||
date(2026, 3, 13), # Friday
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
shadow = await make_event(
|
||||
date(2026, 3, 16), # the next business day
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
|
||||
matched, unmatched = await run()
|
||||
|
||||
assert (matched, unmatched) == (1, 0)
|
||||
assert await matched_ids() == {shadow: api}
|
||||
|
||||
|
||||
async def test_an_exact_date_wins_over_a_next_day_candidate(app):
|
||||
account = await broker_account()
|
||||
instrument = await make_instrument(ticker="GAZP")
|
||||
same_day = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
await make_event(
|
||||
date(2026, 3, 11),
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
shadow = await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
|
||||
await run()
|
||||
|
||||
assert await matched_ids() == {shadow: same_day}
|
||||
|
||||
|
||||
async def test_quantities_must_agree_exactly(app):
|
||||
account = await broker_account()
|
||||
instrument = await make_instrument(ticker="GAZP")
|
||||
await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity="10",
|
||||
price="100",
|
||||
amount="-1000",
|
||||
)
|
||||
await make_event(
|
||||
TRADE_DAY,
|
||||
account_id=account,
|
||||
kind=EventKind.buy,
|
||||
instrument_id=instrument,
|
||||
quantity=Decimal("11"),
|
||||
price="100",
|
||||
amount="-1100",
|
||||
status=EventStatus.shadow,
|
||||
)
|
||||
|
||||
matched, unmatched = await run()
|
||||
|
||||
assert (matched, unmatched) == (0, 1)
|
||||
@@ -0,0 +1,338 @@
|
||||
"""`BrokerEvent` -> `event`: idempotency, pending instruments, shadow status, side effects.
|
||||
|
||||
Every `ParsedReport` here is built by hand: the parsers are a separate contract and a bug in
|
||||
one of them must not be able to fail these tests, which are about what the ledger does with
|
||||
what a parser produced.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from factories import make_instrument
|
||||
from fintracker.db import get_sessionmaker
|
||||
from fintracker.ledger.ingest import ingest
|
||||
from fintracker.ledger.report_import import InstrumentSpec, resolve_pending
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
AccountKind,
|
||||
AccountRole,
|
||||
AssetClass,
|
||||
Event,
|
||||
EventKind,
|
||||
EventSource,
|
||||
EventStatus,
|
||||
Instrument,
|
||||
InstrumentAlias,
|
||||
Lot,
|
||||
PendingInstrument,
|
||||
PendingInstrumentStatus,
|
||||
PriceManual,
|
||||
RawReportFile,
|
||||
RawReportLine,
|
||||
ReportParseStatus,
|
||||
)
|
||||
from fintracker.sources.reports.base import (
|
||||
BrokerEvent,
|
||||
InstrumentRef,
|
||||
ParsedReport,
|
||||
)
|
||||
|
||||
SOURCE = "report_sber"
|
||||
ACCOUNT_NO = "S930W42"
|
||||
UNKNOWN_ISIN = "RU000A1035S8"
|
||||
|
||||
|
||||
async def make_broker_account(
|
||||
*, primary: EventSource | None = EventSource.report_sber, source: str = SOURCE
|
||||
) -> int:
|
||||
async with get_sessionmaker()() as session:
|
||||
account = Account(
|
||||
kind=AccountKind.broker,
|
||||
source=source,
|
||||
source_id=ACCOUNT_NO,
|
||||
name="Сбер ИИС",
|
||||
currency="RUB",
|
||||
role=AccountRole.investment,
|
||||
include_in_net_worth=False,
|
||||
primary_event_source=primary,
|
||||
)
|
||||
session.add(account)
|
||||
await session.commit()
|
||||
return account.id
|
||||
|
||||
|
||||
async def make_file(account_id: int | None = None) -> int:
|
||||
async with get_sessionmaker()() as session:
|
||||
row = RawReportFile(
|
||||
broker="sber",
|
||||
filename="report.html",
|
||||
sha256="0" * 64,
|
||||
size_bytes=10,
|
||||
parser_name=SOURCE,
|
||||
parser_version="1",
|
||||
parse_status=ReportParseStatus.parsed,
|
||||
account_id=account_id,
|
||||
)
|
||||
session.add(row)
|
||||
await session.commit()
|
||||
return row.id
|
||||
|
||||
|
||||
def unknown_ref() -> InstrumentRef:
|
||||
return InstrumentRef(
|
||||
isin=UNKNOWN_ISIN,
|
||||
ticker="STME",
|
||||
name="Первая-ВечныйПортф БПИФ",
|
||||
currency="RUB",
|
||||
asset_class_hint="fund",
|
||||
source_key=f"ISIN:{UNKNOWN_ISIN}",
|
||||
)
|
||||
|
||||
|
||||
def report(
|
||||
events: list[BrokerEvent],
|
||||
*,
|
||||
meta: dict | None = None,
|
||||
instruments: list[InstrumentRef] | None = None,
|
||||
) -> ParsedReport:
|
||||
return ParsedReport(
|
||||
broker="sber",
|
||||
account_external_id=ACCOUNT_NO,
|
||||
period_from=date(2026, 2, 11),
|
||||
period_to=date(2026, 9, 17),
|
||||
parser_version="1",
|
||||
events=events,
|
||||
instruments=instruments or [],
|
||||
meta=meta or {},
|
||||
)
|
||||
|
||||
|
||||
def buy(
|
||||
ref: InstrumentRef | None,
|
||||
*,
|
||||
trade_no: str | None = "15678045077",
|
||||
d: date = date(2026, 2, 24),
|
||||
qty: str = "10",
|
||||
price: str = "100",
|
||||
line_no: int = 1,
|
||||
) -> BrokerEvent:
|
||||
return BrokerEvent(
|
||||
kind=EventKind.buy,
|
||||
trade_date=d,
|
||||
settle_date=d,
|
||||
amount=Decimal(qty) * Decimal(price) * -1,
|
||||
currency="RUB",
|
||||
instrument=ref,
|
||||
quantity=Decimal(qty),
|
||||
price=Decimal(price),
|
||||
price_currency="RUB",
|
||||
fee=Decimal("0.39"),
|
||||
trade_no=trade_no,
|
||||
description="Покупка",
|
||||
raw_line_no=line_no,
|
||||
)
|
||||
|
||||
|
||||
async def run_ingest(account_id: int, parsed: ParsedReport, *, file_id: int | None = None):
|
||||
async with get_sessionmaker()() as session:
|
||||
account = await session.get(Account, account_id)
|
||||
assert account is not None
|
||||
result = await ingest(
|
||||
session, account=account, parsed=parsed, source=SOURCE, file_id=file_id
|
||||
)
|
||||
await session.commit()
|
||||
return result
|
||||
|
||||
|
||||
async def count(model, *where) -> int:
|
||||
async with get_sessionmaker()() as session:
|
||||
return await session.scalar(select(func.count()).select_from(model).where(*where)) or 0
|
||||
|
||||
|
||||
async def test_same_report_twice_creates_nothing_the_second_time(app):
|
||||
account_id = await make_broker_account()
|
||||
instrument_id = await make_instrument(ticker="GAZP", name="Газпром")
|
||||
ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
|
||||
parsed = report([buy(ref)])
|
||||
|
||||
first = await run_ingest(account_id, parsed)
|
||||
second = await run_ingest(account_id, parsed)
|
||||
|
||||
assert first.events_created == 1
|
||||
assert second.events_created == 0
|
||||
assert second.events_updated == 1
|
||||
assert second.events_duplicate == 1
|
||||
assert await count(Event) == 1
|
||||
async with get_sessionmaker()() as session:
|
||||
event = (await session.execute(select(Event))).scalar_one()
|
||||
assert event.instrument_id == instrument_id
|
||||
assert event.status == EventStatus.confirmed
|
||||
|
||||
|
||||
async def test_unknown_isin_parks_the_instrument_and_counts_repeats(app):
|
||||
account_id = await make_broker_account()
|
||||
parsed = report([buy(unknown_ref())])
|
||||
|
||||
first = await run_ingest(account_id, parsed)
|
||||
assert first.events_pending == 1
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
event = (await session.execute(select(Event))).scalar_one()
|
||||
pending = (await session.execute(select(PendingInstrument))).scalar_one()
|
||||
assert event.status == EventStatus.pending
|
||||
assert event.instrument_id is None
|
||||
assert (event.meta or {})["pending_key"] == f"ISIN:{UNKNOWN_ISIN}"
|
||||
assert pending.source == SOURCE
|
||||
assert pending.isin == UNKNOWN_ISIN
|
||||
assert pending.occurrences == 1
|
||||
|
||||
await run_ingest(account_id, parsed)
|
||||
async with get_sessionmaker()() as session:
|
||||
rows = (await session.execute(select(PendingInstrument))).scalars().all()
|
||||
assert len(rows) == 1
|
||||
assert rows[0].occurrences == 2
|
||||
|
||||
|
||||
async def test_resolving_a_pending_instrument_binds_and_confirms_its_events(app):
|
||||
account_id = await make_broker_account()
|
||||
await run_ingest(account_id, report([buy(unknown_ref())]))
|
||||
instrument_id = await make_instrument(ticker="STME", name="БПИФ", board="TQTF")
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
pending = (await session.execute(select(PendingInstrument))).scalar_one()
|
||||
outcome = await resolve_pending(
|
||||
session, pending, action="link", instrument_id=instrument_id
|
||||
)
|
||||
assert outcome.events_bound == 1
|
||||
assert outcome.alias_created is True
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
event = (await session.execute(select(Event))).scalar_one()
|
||||
alias = (await session.execute(select(InstrumentAlias))).scalar_one()
|
||||
lots = (await session.execute(select(Lot))).scalars().all()
|
||||
pending = (await session.execute(select(PendingInstrument))).scalar_one()
|
||||
assert event.status == EventStatus.confirmed
|
||||
assert event.instrument_id == instrument_id
|
||||
assert "pending_key" not in (event.meta or {})
|
||||
assert (alias.source, alias.source_key) == (SOURCE, f"ISIN:{UNKNOWN_ISIN}")
|
||||
assert pending.status == PendingInstrumentStatus.resolved
|
||||
assert [(lot.instrument_id, lot.qty_open) for lot in lots] == [(instrument_id, Decimal(10))]
|
||||
|
||||
|
||||
async def test_report_on_an_api_primary_account_lands_as_shadow(app):
|
||||
account_id = await make_broker_account(primary=EventSource.tinvest_api, source="tinvest")
|
||||
await make_instrument(ticker="GAZP", name="Газпром")
|
||||
ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
|
||||
|
||||
result = await run_ingest(account_id, report([buy(ref)]))
|
||||
|
||||
assert result.events_shadow == 1
|
||||
async with get_sessionmaker()() as session:
|
||||
event = (await session.execute(select(Event))).scalar_one()
|
||||
assert event.status == EventStatus.shadow
|
||||
|
||||
|
||||
async def test_manual_prices_from_meta_become_price_manual_rows(app):
|
||||
account_id = await make_broker_account()
|
||||
instrument_id = await make_instrument(ticker="GAZP", name="Газпром")
|
||||
ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
|
||||
parsed = report(
|
||||
[buy(ref)],
|
||||
meta={
|
||||
"manual_prices": [
|
||||
{
|
||||
"instrument_key": "TICKER:GAZP/TQBR",
|
||||
"d": "2026-09-17",
|
||||
"price": "123.45",
|
||||
"currency": "RUB",
|
||||
}
|
||||
]
|
||||
},
|
||||
)
|
||||
|
||||
result = await run_ingest(account_id, parsed)
|
||||
|
||||
assert result.manual_prices == 1
|
||||
async with get_sessionmaker()() as session:
|
||||
row = (await session.execute(select(PriceManual))).scalar_one()
|
||||
assert row.instrument_id == instrument_id
|
||||
assert row.d == date(2026, 9, 17)
|
||||
assert row.price == Decimal("123.45")
|
||||
|
||||
|
||||
async def test_payout_hint_turns_a_dividend_on_a_bond_into_a_coupon(app):
|
||||
account_id = await make_broker_account()
|
||||
bond_id = await make_instrument(
|
||||
ticker="SU26238", name="ОФЗ 26238", asset_class=AssetClass.bond, board="TQOB"
|
||||
)
|
||||
ref = InstrumentRef(ticker="SU26238", board="TQOB", source_key="TICKER:SU26238/TQOB")
|
||||
payout = BrokerEvent(
|
||||
kind=EventKind.dividend,
|
||||
trade_date=date(2026, 5, 20),
|
||||
amount=Decimal("175.30"),
|
||||
currency="RUB",
|
||||
instrument=ref,
|
||||
trade_no="PAY-1",
|
||||
meta={"payout_hint": "coupon"},
|
||||
raw_line_no=7,
|
||||
)
|
||||
|
||||
await run_ingest(account_id, report([payout]))
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
event = (await session.execute(select(Event))).scalar_one()
|
||||
assert event.kind == EventKind.coupon
|
||||
assert event.instrument_id == bond_id
|
||||
|
||||
|
||||
async def test_raw_report_lines_are_linked_and_carry_no_float(app):
|
||||
account_id = await make_broker_account()
|
||||
await make_instrument(ticker="GAZP", name="Газпром")
|
||||
ref = InstrumentRef(ticker="GAZP", board="TQBR", source_key="TICKER:GAZP/TQBR")
|
||||
file_id = await make_file(account_id)
|
||||
|
||||
await run_ingest(account_id, report([buy(ref, line_no=3)]), file_id=file_id)
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
line = (await session.execute(select(RawReportLine))).scalar_one()
|
||||
event = (await session.execute(select(Event))).scalar_one()
|
||||
assert line.file_id == file_id
|
||||
assert line.line_no == 3
|
||||
assert line.event_id == event.id
|
||||
assert line.dedupe_key == event.dedupe_key
|
||||
assert line.payload["quantity"] == "10"
|
||||
assert _floats(line.payload) == []
|
||||
|
||||
|
||||
def _floats(value, path: str = "$") -> list[str]:
|
||||
"""Every float hiding in a JSONB payload, by path — money must never be one."""
|
||||
if isinstance(value, float):
|
||||
return [path]
|
||||
if isinstance(value, dict):
|
||||
return [p for k, v in value.items() for p in _floats(v, f"{path}.{k}")]
|
||||
if isinstance(value, list):
|
||||
return [p for i, v in enumerate(value) for p in _floats(v, f"{path}[{i}]")]
|
||||
return []
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
("spec", "expected"),
|
||||
[(InstrumentSpec(asset_class="fund", name="X"), AssetClass.fund)],
|
||||
)
|
||||
async def test_create_mode_builds_the_instrument_from_the_report(app, spec, expected):
|
||||
account_id = await make_broker_account()
|
||||
await run_ingest(account_id, report([buy(unknown_ref())]))
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
pending = (await session.execute(select(PendingInstrument))).scalar_one()
|
||||
outcome = await resolve_pending(session, pending, action="create", instrument=spec)
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
instrument = await session.get(Instrument, outcome.instrument_id)
|
||||
assert instrument is not None
|
||||
assert instrument.asset_class == expected
|
||||
@@ -0,0 +1,384 @@
|
||||
"""Сквозной импорт настоящего отчёта: файл → реестр → парсер → леджер → сверка.
|
||||
|
||||
Все остальные тесты фазы 3 честно изолированы: парсеры проверяются на фикстурах без БД,
|
||||
`ingest` — на `ParsedReport`, собранном руками. Это правильно, но между ними остаётся щель,
|
||||
в которую проваливаются ровно те ошибки, ради которых фаза затевалась: парсер отдаёт
|
||||
безупречный `ParsedReport`, ingest безупречно его пишет, а вместе они дают задвоенный
|
||||
леджер, потому что ключи считаются от того, что различается между двумя выгрузками.
|
||||
|
||||
Поэтому здесь ни одного собранного вручную объекта — только байты обезличенных отчётов,
|
||||
`registry.pick` и публичный путь `upload → commit`. Проверки те же, что в плане §Фаза 3:
|
||||
тот же файл дважды даёт ноль новых событий; перекрывающиеся периоды дают каждую сделку по
|
||||
разу; закрывающие позиции и остаток денег из отчёта сходятся с derived; неизвестный ISIN
|
||||
уходит в `pending_instrument`, а после резолва лоты пересобираются.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
from sqlalchemy import func, select
|
||||
|
||||
from fintracker.db import get_sessionmaker
|
||||
from fintracker.ledger.report_import import (
|
||||
InstrumentSpec,
|
||||
build_preview,
|
||||
commit,
|
||||
resolve_pending,
|
||||
upload,
|
||||
)
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
AccountKind,
|
||||
AccountRole,
|
||||
AssetClass,
|
||||
Broker,
|
||||
Event,
|
||||
EventKind,
|
||||
EventSource,
|
||||
EventStatus,
|
||||
Instrument,
|
||||
Lot,
|
||||
PendingInstrument,
|
||||
PendingInstrumentStatus,
|
||||
RawReportFile,
|
||||
)
|
||||
from fintracker.sources.reports import registry
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports" / "sber"
|
||||
FULL = FIXTURES / "S930W42_11022026_17092026.html"
|
||||
AUGUST = FIXTURES / "S930W42_01082026_31082026.html"
|
||||
|
||||
ACCOUNT_NO = "S930W42"
|
||||
|
||||
#: Справочник ценных бумаг полного отчёта: ISIN → (тикер, класс). Заводится заранее, чтобы
|
||||
#: сверка закрывающих позиций проверяла сам импорт, а не резолв инструментов.
|
||||
SECURITIES = {
|
||||
"RU0009062285": ("AFLT", AssetClass.share),
|
||||
"RU0009024277": ("LKOH", AssetClass.share),
|
||||
"RU000A0JR4A1": ("MOEX", AssetClass.share),
|
||||
"RU0008958863": ("MSNG", AssetClass.share),
|
||||
"RU0007775219": ("MTSS", AssetClass.share),
|
||||
"RU000A1035S8": ("STME", AssetClass.etf),
|
||||
"RU0009029540": ("SBER", AssetClass.share),
|
||||
"RU0009046510": ("CHMF", AssetClass.share),
|
||||
"RU0009033591": ("TATN", AssetClass.share),
|
||||
"RU000A100P44": ("SBRB", AssetClass.etf),
|
||||
"RU000A0JRKT8": ("PHOR", AssetClass.share),
|
||||
}
|
||||
|
||||
|
||||
async def make_sber_account(*, primary: EventSource | None = EventSource.report_sber) -> int:
|
||||
async with get_sessionmaker()() as session:
|
||||
account = Account(
|
||||
kind=AccountKind.broker,
|
||||
source="report_sber",
|
||||
source_id=ACCOUNT_NO,
|
||||
broker=Broker.sber,
|
||||
name="Сбер ИИС",
|
||||
currency="RUB",
|
||||
role=AccountRole.investment,
|
||||
primary_event_source=primary,
|
||||
)
|
||||
session.add(account)
|
||||
await session.commit()
|
||||
return account.id
|
||||
|
||||
|
||||
async def make_securities(skip: str | None = None) -> dict[str, int]:
|
||||
"""Инструменты из справочника отчёта. `skip` оставляет один ISIN неизвестным."""
|
||||
ids: dict[str, int] = {}
|
||||
async with get_sessionmaker()() as session:
|
||||
for isin, (ticker, asset_class) in SECURITIES.items():
|
||||
if isin == skip:
|
||||
continue
|
||||
instrument = Instrument(
|
||||
asset_class=asset_class,
|
||||
isin=isin,
|
||||
ticker=ticker,
|
||||
board="TQBR",
|
||||
name=ticker,
|
||||
currency="RUB",
|
||||
)
|
||||
session.add(instrument)
|
||||
await session.flush()
|
||||
ids[isin] = instrument.id
|
||||
await session.commit()
|
||||
return ids
|
||||
|
||||
|
||||
async def import_file(path: Path, account_id: int):
|
||||
"""Полный публичный путь: загрузка, парсинг, commit."""
|
||||
async with get_sessionmaker()() as session:
|
||||
outcome = await upload(
|
||||
session, data=path.read_bytes(), filename=path.name, account_id=account_id
|
||||
)
|
||||
await session.commit()
|
||||
file_id = outcome.file.id
|
||||
duplicate_of_id = outcome.duplicate_of_id
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
row = await session.get(RawReportFile, file_id)
|
||||
assert row is not None
|
||||
result = await commit(session, row, account_id=account_id)
|
||||
await session.commit()
|
||||
return result, duplicate_of_id
|
||||
|
||||
|
||||
async def count_events(**where) -> int:
|
||||
async with get_sessionmaker()() as session:
|
||||
stmt = select(func.count()).select_from(Event)
|
||||
for column, value in where.items():
|
||||
stmt = stmt.where(getattr(Event, column) == value)
|
||||
return (await session.execute(stmt)).scalar_one()
|
||||
|
||||
|
||||
# --- 1. формат доезжает до парсера через реестр --------------------------------------------
|
||||
|
||||
|
||||
async def test_upload_routes_the_file_to_the_sber_parser(app) -> None:
|
||||
account_id = await make_sber_account()
|
||||
async with get_sessionmaker()() as session:
|
||||
outcome = await upload(
|
||||
session, data=FULL.read_bytes(), filename=FULL.name, account_id=account_id
|
||||
)
|
||||
await session.commit()
|
||||
row = outcome.file
|
||||
|
||||
assert row.parser_name == "report_sber"
|
||||
assert row.broker == "sber"
|
||||
assert row.account_external_id == ACCOUNT_NO
|
||||
assert row.period_from is not None and row.period_to is not None
|
||||
assert (row.period_from.isoformat(), row.period_to.isoformat()) == (
|
||||
"2026-02-11",
|
||||
"2026-09-17",
|
||||
)
|
||||
assert registry.pick(FULL.read_bytes(), FULL.name) is not None
|
||||
|
||||
|
||||
async def test_upload_writes_no_events(app) -> None:
|
||||
"""Загрузка — это диагностика, а не запись: леджер меняет только commit."""
|
||||
account_id = await make_sber_account()
|
||||
await make_securities()
|
||||
async with get_sessionmaker()() as session:
|
||||
await upload(session, data=FULL.read_bytes(), filename=FULL.name, account_id=account_id)
|
||||
await session.commit()
|
||||
assert await count_events() == 0
|
||||
|
||||
|
||||
# --- 2. тот же файл дважды → ноль новых событий --------------------------------------------
|
||||
|
||||
|
||||
async def test_the_same_file_twice_adds_nothing(app) -> None:
|
||||
account_id = await make_sber_account()
|
||||
await make_securities()
|
||||
|
||||
first, duplicate = await import_file(FULL, account_id)
|
||||
assert duplicate is None
|
||||
assert first.events_created == 35
|
||||
after_first = await count_events()
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
again = await upload(
|
||||
session, data=FULL.read_bytes(), filename=FULL.name, account_id=account_id
|
||||
)
|
||||
await session.commit()
|
||||
assert again.duplicate_of_id is not None, "второй sha256 создал новый импорт"
|
||||
|
||||
assert await count_events() == after_first == 35
|
||||
|
||||
|
||||
# --- 3. перекрывающиеся периоды: каждая сделка ровно один раз -------------------------------
|
||||
|
||||
|
||||
async def test_overlapping_reports_record_each_trade_once(app) -> None:
|
||||
"""Август целиком входит в полный отчёт: второй импорт не должен ничего добавить.
|
||||
|
||||
Это и есть проверка §1.6 A на живых данных — ключи считаются от номера сделки и от
|
||||
экономического отпечатка операции, а не от того, каким файлом её принесли.
|
||||
"""
|
||||
account_id = await make_sber_account()
|
||||
await make_securities()
|
||||
|
||||
await import_file(FULL, account_id)
|
||||
total_after_full = await count_events()
|
||||
|
||||
august_result, _ = await import_file(AUGUST, account_id)
|
||||
|
||||
assert august_result.events_created == 0, "август задвоил операции полного отчёта"
|
||||
assert august_result.events_updated == 3
|
||||
assert await count_events() == total_after_full
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
keys = (await session.execute(select(Event.dedupe_key))).scalars().all()
|
||||
assert len(keys) == len(set(keys))
|
||||
|
||||
|
||||
async def test_august_first_then_full_history(app) -> None:
|
||||
"""Обратный порядок: сначала месяц, потом вся история — итог тот же."""
|
||||
account_id = await make_sber_account()
|
||||
await make_securities()
|
||||
|
||||
august_result, _ = await import_file(AUGUST, account_id)
|
||||
assert august_result.events_created == 3
|
||||
|
||||
full_result, _ = await import_file(FULL, account_id)
|
||||
assert full_result.events_created == 32
|
||||
assert full_result.events_updated == 3
|
||||
assert await count_events() == 35
|
||||
|
||||
|
||||
# --- 4. закрывающие позиции и деньги отчёта = derived ---------------------------------------
|
||||
|
||||
|
||||
async def test_closing_positions_and_cash_match_the_ledger(app) -> None:
|
||||
"""Главная проверка фазы: то, что брокер напечатал, совпало с тем, что мы вывели."""
|
||||
account_id = await make_sber_account()
|
||||
await make_securities()
|
||||
await import_file(FULL, account_id)
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
row = (
|
||||
(await session.execute(select(RawReportFile).order_by(RawReportFile.id.desc())))
|
||||
.scalars()
|
||||
.first()
|
||||
)
|
||||
assert row is not None
|
||||
preview = await build_preview(session, row)
|
||||
|
||||
mismatched = [p for p in preview.reconciliation.positions if not p.matches]
|
||||
assert not mismatched, [
|
||||
(p.ticker or p.instrument_name, str(p.qty_report), str(p.qty_derived)) for p in mismatched
|
||||
]
|
||||
|
||||
rub = next(c for c in preview.reconciliation.cash if c.currency == "RUB")
|
||||
assert rub.balance_report == Decimal("3171.34")
|
||||
assert rub.balance_derived == rub.balance_report
|
||||
assert preview.reconciliation.matches
|
||||
|
||||
|
||||
async def test_derived_position_equals_the_reports_own_quantity(app) -> None:
|
||||
"""Та же сверка, но из самого леджера: Σ `lot.qty_remaining` против отчёта."""
|
||||
account_id = await make_sber_account()
|
||||
ids = await make_securities()
|
||||
await import_file(FULL, account_id)
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
rows = (
|
||||
await session.execute(
|
||||
select(Lot.instrument_id, func.sum(Lot.qty_remaining)).group_by(Lot.instrument_id)
|
||||
)
|
||||
).all()
|
||||
held: dict[int, Decimal] = {instrument_id: qty for instrument_id, qty in rows}
|
||||
|
||||
# «Портфель Ценных Бумаг» полного отчёта, колонка «Конец периода / Количество, шт»
|
||||
assert held[ids["RU0009062285"]] == Decimal("130") # Аэрофлот
|
||||
assert held[ids["RU0008958863"]] == Decimal("3000") # Мосэнерго
|
||||
assert held[ids["RU0009029540"]] == Decimal("20") # Сбербанк
|
||||
assert held.get(ids["RU000A1035S8"], Decimal(0)) == 0 # STME куплен и продан целиком
|
||||
assert held.get(ids["RU000A100P44"], Decimal(0)) == 0 # SBRB тоже закрыт
|
||||
|
||||
|
||||
# --- 5. неизвестный ISIN → pending → резолв → лоты ------------------------------------------
|
||||
|
||||
|
||||
async def test_unknown_isin_parks_and_resolves(app) -> None:
|
||||
"""Инструмент не угадывается: события ждут, пока его подтвердят, и только тогда считаются."""
|
||||
account_id = await make_sber_account()
|
||||
ids = await make_securities(skip="RU0009062285") # Аэрофлот остаётся неизвестным
|
||||
|
||||
result, _ = await import_file(FULL, account_id)
|
||||
assert result.pending_instruments >= 1
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
pending = (
|
||||
(
|
||||
await session.execute(
|
||||
select(PendingInstrument).where(
|
||||
PendingInstrument.status == PendingInstrumentStatus.pending
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
assert [p.isin for p in pending] == ["RU0009062285"]
|
||||
assert pending[0].occurrences >= 2 # Аэрофлот куплен двумя сделками
|
||||
pending_id = pending[0].id
|
||||
|
||||
assert await count_events(status=EventStatus.pending) >= 2
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
row = await session.get(PendingInstrument, pending_id)
|
||||
assert row is not None
|
||||
outcome = await resolve_pending(
|
||||
session,
|
||||
row,
|
||||
action="create",
|
||||
instrument=InstrumentSpec(
|
||||
asset_class="share",
|
||||
isin="RU0009062285",
|
||||
ticker="AFLT",
|
||||
board="TQBR",
|
||||
name="Аэрофлот",
|
||||
currency="RUB",
|
||||
),
|
||||
)
|
||||
await session.commit()
|
||||
assert outcome.events_bound >= 2
|
||||
|
||||
assert await count_events(status=EventStatus.pending) == 0
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
instrument = (
|
||||
await session.execute(select(Instrument).where(Instrument.isin == "RU0009062285"))
|
||||
).scalar_one()
|
||||
qty = (
|
||||
await session.execute(
|
||||
select(func.sum(Lot.qty_remaining)).where(Lot.instrument_id == instrument.id)
|
||||
)
|
||||
).scalar_one()
|
||||
assert qty == Decimal("130"), "после резолва лоты не пересобрались"
|
||||
|
||||
assert ids # остальные инструменты были известны заранее
|
||||
|
||||
|
||||
# --- 6. чужой источник пишется тенью -------------------------------------------------------
|
||||
|
||||
|
||||
async def test_report_on_an_api_driven_account_lands_as_shadow(app) -> None:
|
||||
"""У счёта один primary_event_source; отчёт поверх API — evidence, а не леджер."""
|
||||
account_id = await make_sber_account(primary=EventSource.tinvest_api)
|
||||
await make_securities()
|
||||
|
||||
result, _ = await import_file(FULL, account_id)
|
||||
|
||||
assert result.events_shadow == 35
|
||||
assert await count_events(status=EventStatus.confirmed) == 0
|
||||
assert await count_events(status=EventStatus.shadow) == 35
|
||||
|
||||
|
||||
# --- 7. трассируемость ---------------------------------------------------------------------
|
||||
|
||||
|
||||
async def test_every_event_points_back_at_its_raw_line(app) -> None:
|
||||
"""`raw_report_line` — то, по чему через полгода восстанавливают, откуда взялось число."""
|
||||
account_id = await make_sber_account()
|
||||
await make_securities()
|
||||
await import_file(FULL, account_id)
|
||||
|
||||
async with get_sessionmaker()() as session:
|
||||
linked = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(Event).where(Event.raw_ref.is_not(None))
|
||||
)
|
||||
).scalar_one()
|
||||
assert linked == 35
|
||||
|
||||
commission = (
|
||||
await session.execute(
|
||||
select(func.count()).select_from(Event).where(Event.kind == EventKind.commission)
|
||||
)
|
||||
).scalar_one()
|
||||
assert commission == 0, "комиссия Сбера должна быть капитализирована в сделку"
|
||||
Reference in New Issue
Block a user