feat(reports): протокол и парсеры отчётов Сбера, ВТБ и Snowball CSV

sources/reports/base.py — контракт ReportParser (плагины: sniff/parse, чистые функции без БД). registry.py выбирает парсер по содержимому файла, CSV последним: он узнаёт файл по набору колонок и иначе перехватил бы чужой формат. Комиссия капитализируется в сделку, отдельным событием не эмитится: и Сбер, и ВТБ печатают её дважды — колонками в сделках и строками в движении денег, суммы совпадают, второе прочтение задвоило бы её. Расчётные строки («Сделка от …», «Сальдо расчетов по сделкам») не эмитятся — это денежные ноги уже учтённых сделок. У Сбера таблица «Информация о зачислениях на ИИС» кумулятивна за календарный год и в леджер не идёт, иначе два пересекающихся отчёта задвоили бы пополнения. CSV Snowball сводный по всем брокерам — годится как сверка потоков и сделок на уровне портфеля, но не позиций по счетам; CUSTOM_HOLDING_PRICE не событие, а цена — уходит в meta для price_manual. Обезличивание — anonymize.py + scripts/anonymize_reports.py, секреты собираются по всему корпусу отчётов разом (Snowball цитирует номер договора Сбера в примечании к переводу). test_fixtures_anonymized.py падает, если в tests/fixtures/reports/ вне raw/ найдётся ИНН, ФИО или номер счёта — и по форме (работает в CI без raw/), и по фактическому содержимому raw/, когда оно на месте.
2026-09-19 10:39:37 +03:00
parent 6b32c79e79
commit 2e742a093b
25 changed files with 7284 additions and 0 deletions
@@ -0,0 +1,410 @@
+"""VTB xlsx report parser against the two anonymised fixtures.
+
+The fixtures are the same account rendered twice: the whole life of the IIS (12.02–17.09) and
+the single month of August, which is a strict subset of it. That overlap is the point — it is
+exactly the case the dedupe key exists for, so the deal both files contain must produce one
+key, and the monthly file must not invent a trade the full one does not have.
+
+Every number asserted here is taken from «Сводная информация по субсчёту Клиента», the
+broker's own totals block: the parser is trusted only insofar as what it extracted adds up to
+what VTB printed.
+"""
+
+from __future__ import annotations
+
+import io
+import zipfile
+from collections import Counter
+from dataclasses import fields, is_dataclass
+from decimal import Decimal
+from pathlib import Path
+
+import pytest
+from openpyxl import load_workbook
+
+from fintracker.models.ledger import EventKind
+from fintracker.sources.reports.base import ParsedReport
+from fintracker.sources.reports.vtb import common
+from fintracker.sources.reports.vtb.xlsx import VtbXlsxParser
+
+FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports"
+FULL = FIXTURES / "vtb" / "reportd0235f1e-55ac-fdc8-5c7b-279e4fe26e2b.xlsx"
+AUGUST = FIXTURES / "vtb" / "report8ef8347a-2deb-e880-959a-d9a8ca12f735.xlsx"
+
+#: The August deal, as printed in both files: «№ сделки» B17476399721, 33 EQMX at 122.9.
+AUGUST_TRADE_NO = "B17476399721"
+
+
+@pytest.fixture(scope="module")
+def parser() -> VtbXlsxParser:
+    return VtbXlsxParser()
+
+
+@pytest.fixture(scope="module")
+def full(parser: VtbXlsxParser) -> ParsedReport:
+    return parser.parse(FULL.read_bytes(), FULL.name)
+
+
+@pytest.fixture(scope="module")
+def august(parser: VtbXlsxParser) -> ParsedReport:
+    return parser.parse(AUGUST.read_bytes(), AUGUST.name)
+
+
+def trades(report: ParsedReport) -> list:
+    return [event for event in report.events if event.kind in {EventKind.buy, EventKind.sell}]
+
+
+def count_security_movement_rows(path: Path) -> int:
+    """Rows of «Движение ценных бумаг» counted straight off the sheet.
+
+    Deliberately independent of the parser: the section is the settlement side of the same
+    deals, so its row count is an outside check that no trade was dropped or doubled.
+    """
+    workbook = load_workbook(io.BytesIO(path.read_bytes()), read_only=True, data_only=True)
+    sheet = workbook["brokerage_report"]
+    sheet.reset_dimensions()
+    inside = False
+    rows = 0
+    for row in sheet.iter_rows(values_only=True):
+        label = common.normalize_text(row[1] if len(row) > 1 else None)
+        if label == "движение ценных бумаг":
+            inside = True
+            continue
+        if not inside:
+            continue
+        if label in {"заключенные в отчетном периоде сделки с ценными бумагами"}:
+            break
+        if "расчеты по заключенным сделкам" in common.normalize_text(
+            row[16] if len(row) > 16 else None
+        ):
+            rows += 1
+    workbook.close()
+    return rows
+
+
+# -- 1. sniff ----------------------------------------------------------------------------
+
+
+def test_sniff_accepts_both_vtb_files(parser: VtbXlsxParser) -> None:
+    for path in (FULL, AUGUST):
+        assert parser.sniff(path.read_bytes(), path.name) is True
+
+
+def test_sniff_rejects_other_brokers(parser: VtbXlsxParser) -> None:
+    others = list((FIXTURES / "sber").glob("*.html")) + list((FIXTURES / "snowball").glob("*.csv"))
+    assert others, "фикстуры других брокеров не найдены — проверка бессмысленна"
+    for path in others:
+        assert parser.sniff(path.read_bytes(), path.name) is False
+
+
+def test_sniff_never_raises_on_garbage(parser: VtbXlsxParser) -> None:
+    """A registry walks every parser over every upload, so a False must never be an
+    exception — including for a zip that merely looks like a workbook."""
+    buffer = io.BytesIO()
+    with zipfile.ZipFile(buffer, "w") as archive:
+        archive.writestr("xl/workbook.xml", "<workbook/>")
+    for payload in (b"", b"not a zip", b"PK\x03\x04broken", buffer.getvalue()):
+        assert parser.sniff(payload, "whatever.xlsx") is False
+
+
+def test_sniff_ignores_the_file_name(parser: VtbXlsxParser) -> None:
+    """The name is a GUID with no format in it; content decides."""
+    assert parser.sniff(FULL.read_bytes(), "totally-unrelated.bin") is True
+
+
+# -- 2. the full report ------------------------------------------------------------------
+
+
+def test_full_report_header(full: ParsedReport) -> None:
+    assert full.broker == "vtb"
+    assert full.account_external_id == "77R593"
+    assert (full.period_from.isoformat(), full.period_to.isoformat()) == (
+        "2026-02-12",
+        "2026-09-17",
+    )
+    assert full.meta["iis_opened_at"] == "2026-02-12"
+
+
+def test_full_report_trades_come_only_from_concluded_section(full: ParsedReport) -> None:
+    """«Завершенные» repeats «Заключенные» with the same deal numbers, and «Движение ценных
+    бумаг» settles those same deals — so ten deals must yield ten events, not twenty or
+    thirty."""
+    numbers = [event.trade_no for event in trades(full)]
+    assert len(numbers) == len(set(numbers)) == count_security_movement_rows(FULL) == 10
+
+
+def test_full_report_parses_without_warnings(full: ParsedReport) -> None:
+    assert full.warnings == []
+
+
+# -- 3. money ----------------------------------------------------------------------------
+
+
+def test_full_report_deposits_and_fees(full: ParsedReport) -> None:
+    deposits = [event for event in full.events if event.kind is EventKind.deposit]
+    assert sum(event.amount for event in deposits) == Decimal("26100")
+    assert all(event.amount > 0 for event in deposits), "пополнение увеличивает счёт"
+
+    # The per-deal commissions add up to «Вознаграждение брокера» of the summary block to the
+    # kopeck, which is why they are capitalised into the deals instead of being emitted
+    # separately: `ledger/lots.py` already puts `fee` into the cost of the lot.
+    assert sum((event.fee or Decimal(0)) for event in full.events) == Decimal("3.64")
+    assert Decimal(full.meta["summary"]["вознаграждение брокера"]) == Decimal("-3.64")
+    assert not [event for event in full.events if event.kind is EventKind.commission]
+
+
+def test_full_report_signs(full: ParsedReport) -> None:
+    for event in trades(full):
+        assert event.quantity is not None
+        if event.kind is EventKind.buy:
+            assert event.quantity > 0 and event.amount < 0
+        else:
+            assert event.quantity < 0 and event.amount > 0
+
+
+def test_settlement_saldo_is_not_emitted_twice(full: ParsedReport) -> None:
+    """«Сальдо расчетов по сделкам с ценными бумагами» is the money leg of the trades; the
+    cash section must contribute deposits only."""
+    cash_kinds = Counter(event.kind for event in full.events if event.meta.get("section") == "cash")
+    assert cash_kinds == Counter({EventKind.deposit: 8})
+
+
+def test_closing_cash_follows_from_the_events(full: ParsedReport) -> None:
+    """The whole-file check: opening + everything emitted = closing, as VTB states it."""
+    summary = full.meta["summary"]
+    opening = Decimal(summary["входящий остаток денежных средств"])
+    closing = Decimal(summary["исходящий остаток денежных средств"])
+    assert opening + sum(event.amount for event in full.events) == closing == Decimal("120.99")
+
+
+# -- 4. positions and cash balances -------------------------------------------------------
+
+
+def test_full_report_positions_end(full: ParsedReport) -> None:
+    positions = {position.instrument.isin: position for position in full.positions_end}
+    assert set(positions) == {"RU000A101EJ5", "RU000A1014L8"}
+    assert positions["RU000A101EJ5"].qty == Decimal("197")
+    assert positions["RU000A1014L8"].qty == Decimal("0")
+    eqmx = positions["RU000A101EJ5"].instrument
+    assert (eqmx.ticker, eqmx.asset_class_hint) == ("EQMX", "etf")
+    assert eqmx.name == "EQMX ETF"
+    assert eqmx.meta["reg_number"] == "3965"
+    assert positions["RU000A101EJ5"].price == Decimal("124.653")
+
+
+def test_currency_is_normalised_to_iso(full: ParsedReport) -> None:
+    """VTB prints the pre-ISO `RUR`; the ledger stores three-letter ISO codes only."""
+    assert [(cash.currency, cash.balance) for cash in full.cash_end] == [("RUB", Decimal("120.99"))]
+    codes = {cash.currency for cash in full.cash_end}
+    codes |= {position.currency for position in full.positions_end}
+    codes |= {event.currency for event in full.events}
+    codes |= {event.price_currency for event in trades(full)}
+    assert codes == {"RUB"}
+
+
+def test_instrument_hint_reaches_trade_events(full: ParsedReport) -> None:
+    """Only the positions table names the asset class («ПАЙ»); the trades table does not, so
+    the hint is carried over by ISIN."""
+    assert {ref.asset_class_hint for ref in full.instruments} == {"etf"}
+    assert all(
+        event.instrument is not None and event.instrument.asset_class_hint == "etf"
+        for event in trades(full)
+    )
+
+
+# -- 5. the monthly report and the cross-file dedupe key ----------------------------------
+
+
+def test_august_report(august: ParsedReport) -> None:
+    assert (august.period_from.isoformat(), august.period_to.isoformat()) == (
+        "2026-08-01",
+        "2026-08-31",
+    )
+    assert august.account_external_id == "77R593"
+    assert august.warnings == []
+
+    deals = trades(august)
+    assert len(deals) == 1
+    deal = deals[0]
+    assert deal.kind is EventKind.buy
+    assert deal.instrument is not None and deal.instrument.isin == "RU000A101EJ5"
+    assert deal.quantity == Decimal("33")
+    assert deal.price == Decimal("122.9")
+    assert deal.trade_no == AUGUST_TRADE_NO
+    # 33 * 122.9 = 4055.70 plus the 0.81 commission capitalised into the deal.
+    assert deal.amount == Decimal("-4056.51")
+    assert deal.fee == Decimal("0.81")
+
+
+def test_same_deal_has_one_dedupe_key_in_both_reports(
+    full: ParsedReport, august: ParsedReport
+) -> None:
+    """The overlapping-period case from plan §1.6 A: re-importing August after the full
+    report must upsert, not duplicate."""
+    keys = {}
+    for report in (full, august):
+        for event in trades(report):
+            if event.trade_no == AUGUST_TRADE_NO:
+                keys[report.period_to] = common.dedupe_key(report.account_external_id, event)
+    assert len(keys) == 2
+    assert len(set(keys.values())) == 1
+
+
+def test_dedupe_key_uses_the_brokers_own_deal_number(full: ParsedReport) -> None:
+    """«№ сделки» (Z), not «№ сделки у организатора торгов» (AC).
+
+    The key is `sha1(broker | account | trade_no)` and is therefore already unique within the
+    broker; what it needs is the number VTB prints in every rendering of its own report. The
+    organiser's number is a different string (`17476399721` against `B17476399721`) and is
+    absent for over-the-counter deals in some formats.
+    """
+    deal = next(event for event in trades(full) if event.trade_no == AUGUST_TRADE_NO)
+    assert deal.meta["exchange_trade_no"] == "17476399721"
+    assert common.dedupe_key("77R593", deal) != common.dedupe_key(
+        "77R593",
+        type(deal)(
+            **{
+                **{field.name: getattr(deal, field.name) for field in fields(deal)},
+                "trade_no": deal.meta["exchange_trade_no"],
+            }
+        ),
+    )
+
+
+def test_cash_rows_get_distinct_fingerprint_keys(full: ParsedReport) -> None:
+    """Deposits carry no deal number and fall back to a fingerprint; two 4 000 ₽ top-ups in
+    different months must still differ."""
+    keys = [
+        common.dedupe_key(full.account_external_id, event)
+        for event in full.events
+        if event.kind is EventKind.deposit
+    ]
+    assert len(set(keys)) == len(keys) == 8
+
+
+# -- 6. concluded vs completed ------------------------------------------------------------
+
+
+def test_concluded_and_completed_deal_numbers_agree(full: ParsedReport) -> None:
+    """Asserted through the parser's own warning channel: a divergence is reported, and for a
+    fully settled period there is none."""
+    assert not [warning for warning in full.warnings if "Завершенных" in warning]
+    parsed = VtbXlsxParser().parse(FULL.read_bytes(), FULL.name)
+    concluded = {event.trade_no for event in trades(parsed)}
+    assert concluded == set(_completed_deal_numbers(FULL))
+
+
+def _completed_deal_numbers(path: Path) -> list[str]:
+    """«№ сделки» of «Завершенные…» read straight off the sheet, bypassing the parser."""
+    workbook = load_workbook(io.BytesIO(path.read_bytes()), read_only=True, data_only=True)
+    sheet = workbook["brokerage_report"]
+    sheet.reset_dimensions()
+    inside = False
+    numbers: list[str] = []
+    for row in sheet.iter_rows(values_only=True):
+        label = common.normalize_text(row[1] if len(row) > 1 else None)
+        if label.startswith("завершенные в отчетном периоде сделки с ценными бумагами"):
+            inside = True
+            continue
+        if not inside:
+            continue
+        value = row[25] if len(row) > 25 else None  # column Z
+        if value and str(value).strip() and common.normalize_text(value) != "№ сделки":
+            numbers.append(str(value).strip())
+    workbook.close()
+    return numbers
+
+
+# -- 7. no floats anywhere ----------------------------------------------------------------
+
+
+def _floats(value: object, path: str = "") -> list[str]:
+    if isinstance(value, bool) or value is None:
+        return []
+    if isinstance(value, float):
+        return [path]
+    if is_dataclass(value) and not isinstance(value, type):
+        found: list[str] = []
+        for field in fields(value):
+            found += _floats(getattr(value, field.name), f"{path}.{field.name}")
+        return found
+    if isinstance(value, dict):
+        return [p for key, item in value.items() for p in _floats(item, f"{path}[{key}]")]
+    if isinstance(value, (list, tuple, set)):
+        return [p for index, item in enumerate(value) for p in _floats(item, f"{path}[{index}]")]
+    return []
+
+
+@pytest.mark.parametrize("name", ["full", "august"])
+def test_no_floats_in_output(name: str, request: pytest.FixtureRequest) -> None:
+    """openpyxl hands back floats for every numeric cell; none of them may survive into the
+    result, or a kopeck disappears into binary rounding on the way to `NUMERIC(24,10)`."""
+    report: ParsedReport = request.getfixturevalue(name)
+    assert _floats(report, name) == []
+
+
+# -- 8. totals against the summary block --------------------------------------------------
+
+
+@pytest.mark.parametrize(("name", "expected"), [("full", "-25975.37"), ("august", "-4055.7")])
+def test_trade_sums_match_the_settlement_saldo(
+    name: str, expected: str, request: pytest.FixtureRequest
+) -> None:
+    """Σ «Сумма сделки» (column M) of the purchases minus the sales equals «Сальдо расчетов по
+    сделкам с ценными бумагами» in the summary.
+
+    The commission has to be taken back out of `amount` to make the comparison: VTB's saldo is
+    the money of the deals themselves, while `event.amount` carries the fee inside it by the
+    ledger's convention. The two agree exactly once that is done — the fee lives in the same
+    file as its own «Вознаграждение брокера» line, and it is that line, not the saldo, that
+    accounts for it.
+    """
+    report: ParsedReport = request.getfixturevalue(name)
+    gross = Decimal(0)
+    for event in trades(report):
+        fee = event.fee or Decimal(0)
+        gross += event.amount + fee if event.kind is EventKind.buy else event.amount - fee
+    assert gross == Decimal(expected)
+    assert gross == Decimal(report.meta["summary"]["сальдо расчетов по сделкам с ценными бумагами"])
+
+
+def test_position_quantities_match_the_settled_movements(full: ParsedReport) -> None:
+    """Closing position = what the trades did, since the account opened empty."""
+    by_isin: dict[str, Decimal] = {}
+    for event in trades(full):
+        assert event.instrument is not None and event.instrument.isin
+        assert event.quantity is not None
+        by_isin[event.instrument.isin] = (
+            by_isin.get(event.instrument.isin, Decimal(0)) + event.quantity
+        )
+    for position in full.positions_end:
+        assert position.instrument.isin is not None
+        assert by_isin[position.instrument.isin] == position.qty
+
+
+# -- parse errors --------------------------------------------------------------------------
+
+
+def test_unreadable_file_raises_parse_error(parser: VtbXlsxParser) -> None:
+    from fintracker.sources.reports.base import ParseError
+
+    with pytest.raises(ParseError):
+        parser.parse(b"PK\x03\x04 definitely not a workbook", "report.xlsx")
+
+
+def test_workbook_without_a_period_raises_parse_error(parser: VtbXlsxParser) -> None:
+    """A file we recognise but cannot place on a timeline must fail loudly rather than import
+    as an empty report."""
+    from openpyxl import Workbook
+
+    from fintracker.sources.reports.base import ParseError
+
+    workbook = Workbook()
+    sheet = workbook.worksheets[0]
+    sheet.title = "brokerage_report"
+    sheet["D1"] = "Отчет Банка ВТБ (ПАО) без периода"
+    buffer = io.BytesIO()
+    workbook.save(buffer)
+
+    with pytest.raises(ParseError):
+        parser.parse(buffer.getvalue(), "report.xlsx")