Dmitry
2e742a093b
sources/reports/base.py — контракт ReportParser (плагины: sniff/parse, чистые функции без БД). registry.py выбирает парсер по содержимому файла, CSV последним: он узнаёт файл по набору колонок и иначе перехватил бы чужой формат. Комиссия капитализируется в сделку, отдельным событием не эмитится: и Сбер, и ВТБ печатают её дважды — колонками в сделках и строками в движении денег, суммы совпадают, второе прочтение задвоило бы её. Расчётные строки («Сделка от …», «Сальдо расчетов по сделкам») не эмитятся — это денежные ноги уже учтённых сделок. У Сбера таблица «Информация о зачислениях на ИИС» кумулятивна за календарный год и в леджер не идёт, иначе два пересекающихся отчёта задвоили бы пополнения. CSV Snowball сводный по всем брокерам — годится как сверка потоков и сделок на уровне портфеля, но не позиций по счетам; CUSTOM_HOLDING_PRICE не событие, а цена — уходит в meta для price_manual. Обезличивание — anonymize.py + scripts/anonymize_reports.py, секреты собираются по всему корпусу отчётов разом (Snowball цитирует номер договора Сбера в примечании к переводу). test_fixtures_anonymized.py падает, если в tests/fixtures/reports/ вне raw/ найдётся ИНН, ФИО или номер счёта — и по форме (работает в CI без raw/), и по фактическому содержимому raw/, когда оно на месте.
411 lines
18 KiB
Python
411 lines
18 KiB
Python
"""VTB xlsx report parser against the two anonymised fixtures.
|
||
|
||
The fixtures are the same account rendered twice: the whole life of the IIS (12.02–17.09) and
|
||
the single month of August, which is a strict subset of it. That overlap is the point — it is
|
||
exactly the case the dedupe key exists for, so the deal both files contain must produce one
|
||
key, and the monthly file must not invent a trade the full one does not have.
|
||
|
||
Every number asserted here is taken from «Сводная информация по субсчёту Клиента», the
|
||
broker's own totals block: the parser is trusted only insofar as what it extracted adds up to
|
||
what VTB printed.
|
||
"""
|
||
|
||
from __future__ import annotations
|
||
|
||
import io
|
||
import zipfile
|
||
from collections import Counter
|
||
from dataclasses import fields, is_dataclass
|
||
from decimal import Decimal
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from openpyxl import load_workbook
|
||
|
||
from fintracker.models.ledger import EventKind
|
||
from fintracker.sources.reports.base import ParsedReport
|
||
from fintracker.sources.reports.vtb import common
|
||
from fintracker.sources.reports.vtb.xlsx import VtbXlsxParser
|
||
|
||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports"
|
||
FULL = FIXTURES / "vtb" / "reportd0235f1e-55ac-fdc8-5c7b-279e4fe26e2b.xlsx"
|
||
AUGUST = FIXTURES / "vtb" / "report8ef8347a-2deb-e880-959a-d9a8ca12f735.xlsx"
|
||
|
||
#: The August deal, as printed in both files: «№ сделки» B17476399721, 33 EQMX at 122.9.
|
||
AUGUST_TRADE_NO = "B17476399721"
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def parser() -> VtbXlsxParser:
|
||
return VtbXlsxParser()
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def full(parser: VtbXlsxParser) -> ParsedReport:
|
||
return parser.parse(FULL.read_bytes(), FULL.name)
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def august(parser: VtbXlsxParser) -> ParsedReport:
|
||
return parser.parse(AUGUST.read_bytes(), AUGUST.name)
|
||
|
||
|
||
def trades(report: ParsedReport) -> list:
|
||
return [event for event in report.events if event.kind in {EventKind.buy, EventKind.sell}]
|
||
|
||
|
||
def count_security_movement_rows(path: Path) -> int:
|
||
"""Rows of «Движение ценных бумаг» counted straight off the sheet.
|
||
|
||
Deliberately independent of the parser: the section is the settlement side of the same
|
||
deals, so its row count is an outside check that no trade was dropped or doubled.
|
||
"""
|
||
workbook = load_workbook(io.BytesIO(path.read_bytes()), read_only=True, data_only=True)
|
||
sheet = workbook["brokerage_report"]
|
||
sheet.reset_dimensions()
|
||
inside = False
|
||
rows = 0
|
||
for row in sheet.iter_rows(values_only=True):
|
||
label = common.normalize_text(row[1] if len(row) > 1 else None)
|
||
if label == "движение ценных бумаг":
|
||
inside = True
|
||
continue
|
||
if not inside:
|
||
continue
|
||
if label in {"заключенные в отчетном периоде сделки с ценными бумагами"}:
|
||
break
|
||
if "расчеты по заключенным сделкам" in common.normalize_text(
|
||
row[16] if len(row) > 16 else None
|
||
):
|
||
rows += 1
|
||
workbook.close()
|
||
return rows
|
||
|
||
|
||
# -- 1. sniff ----------------------------------------------------------------------------
|
||
|
||
|
||
def test_sniff_accepts_both_vtb_files(parser: VtbXlsxParser) -> None:
|
||
for path in (FULL, AUGUST):
|
||
assert parser.sniff(path.read_bytes(), path.name) is True
|
||
|
||
|
||
def test_sniff_rejects_other_brokers(parser: VtbXlsxParser) -> None:
|
||
others = list((FIXTURES / "sber").glob("*.html")) + list((FIXTURES / "snowball").glob("*.csv"))
|
||
assert others, "фикстуры других брокеров не найдены — проверка бессмысленна"
|
||
for path in others:
|
||
assert parser.sniff(path.read_bytes(), path.name) is False
|
||
|
||
|
||
def test_sniff_never_raises_on_garbage(parser: VtbXlsxParser) -> None:
|
||
"""A registry walks every parser over every upload, so a False must never be an
|
||
exception — including for a zip that merely looks like a workbook."""
|
||
buffer = io.BytesIO()
|
||
with zipfile.ZipFile(buffer, "w") as archive:
|
||
archive.writestr("xl/workbook.xml", "<workbook/>")
|
||
for payload in (b"", b"not a zip", b"PK\x03\x04broken", buffer.getvalue()):
|
||
assert parser.sniff(payload, "whatever.xlsx") is False
|
||
|
||
|
||
def test_sniff_ignores_the_file_name(parser: VtbXlsxParser) -> None:
|
||
"""The name is a GUID with no format in it; content decides."""
|
||
assert parser.sniff(FULL.read_bytes(), "totally-unrelated.bin") is True
|
||
|
||
|
||
# -- 2. the full report ------------------------------------------------------------------
|
||
|
||
|
||
def test_full_report_header(full: ParsedReport) -> None:
|
||
assert full.broker == "vtb"
|
||
assert full.account_external_id == "77R593"
|
||
assert (full.period_from.isoformat(), full.period_to.isoformat()) == (
|
||
"2026-02-12",
|
||
"2026-09-17",
|
||
)
|
||
assert full.meta["iis_opened_at"] == "2026-02-12"
|
||
|
||
|
||
def test_full_report_trades_come_only_from_concluded_section(full: ParsedReport) -> None:
|
||
"""«Завершенные» repeats «Заключенные» with the same deal numbers, and «Движение ценных
|
||
бумаг» settles those same deals — so ten deals must yield ten events, not twenty or
|
||
thirty."""
|
||
numbers = [event.trade_no for event in trades(full)]
|
||
assert len(numbers) == len(set(numbers)) == count_security_movement_rows(FULL) == 10
|
||
|
||
|
||
def test_full_report_parses_without_warnings(full: ParsedReport) -> None:
|
||
assert full.warnings == []
|
||
|
||
|
||
# -- 3. money ----------------------------------------------------------------------------
|
||
|
||
|
||
def test_full_report_deposits_and_fees(full: ParsedReport) -> None:
|
||
deposits = [event for event in full.events if event.kind is EventKind.deposit]
|
||
assert sum(event.amount for event in deposits) == Decimal("26100")
|
||
assert all(event.amount > 0 for event in deposits), "пополнение увеличивает счёт"
|
||
|
||
# The per-deal commissions add up to «Вознаграждение брокера» of the summary block to the
|
||
# kopeck, which is why they are capitalised into the deals instead of being emitted
|
||
# separately: `ledger/lots.py` already puts `fee` into the cost of the lot.
|
||
assert sum((event.fee or Decimal(0)) for event in full.events) == Decimal("3.64")
|
||
assert Decimal(full.meta["summary"]["вознаграждение брокера"]) == Decimal("-3.64")
|
||
assert not [event for event in full.events if event.kind is EventKind.commission]
|
||
|
||
|
||
def test_full_report_signs(full: ParsedReport) -> None:
|
||
for event in trades(full):
|
||
assert event.quantity is not None
|
||
if event.kind is EventKind.buy:
|
||
assert event.quantity > 0 and event.amount < 0
|
||
else:
|
||
assert event.quantity < 0 and event.amount > 0
|
||
|
||
|
||
def test_settlement_saldo_is_not_emitted_twice(full: ParsedReport) -> None:
|
||
"""«Сальдо расчетов по сделкам с ценными бумагами» is the money leg of the trades; the
|
||
cash section must contribute deposits only."""
|
||
cash_kinds = Counter(event.kind for event in full.events if event.meta.get("section") == "cash")
|
||
assert cash_kinds == Counter({EventKind.deposit: 8})
|
||
|
||
|
||
def test_closing_cash_follows_from_the_events(full: ParsedReport) -> None:
|
||
"""The whole-file check: opening + everything emitted = closing, as VTB states it."""
|
||
summary = full.meta["summary"]
|
||
opening = Decimal(summary["входящий остаток денежных средств"])
|
||
closing = Decimal(summary["исходящий остаток денежных средств"])
|
||
assert opening + sum(event.amount for event in full.events) == closing == Decimal("120.99")
|
||
|
||
|
||
# -- 4. positions and cash balances -------------------------------------------------------
|
||
|
||
|
||
def test_full_report_positions_end(full: ParsedReport) -> None:
|
||
positions = {position.instrument.isin: position for position in full.positions_end}
|
||
assert set(positions) == {"RU000A101EJ5", "RU000A1014L8"}
|
||
assert positions["RU000A101EJ5"].qty == Decimal("197")
|
||
assert positions["RU000A1014L8"].qty == Decimal("0")
|
||
eqmx = positions["RU000A101EJ5"].instrument
|
||
assert (eqmx.ticker, eqmx.asset_class_hint) == ("EQMX", "etf")
|
||
assert eqmx.name == "EQMX ETF"
|
||
assert eqmx.meta["reg_number"] == "3965"
|
||
assert positions["RU000A101EJ5"].price == Decimal("124.653")
|
||
|
||
|
||
def test_currency_is_normalised_to_iso(full: ParsedReport) -> None:
|
||
"""VTB prints the pre-ISO `RUR`; the ledger stores three-letter ISO codes only."""
|
||
assert [(cash.currency, cash.balance) for cash in full.cash_end] == [("RUB", Decimal("120.99"))]
|
||
codes = {cash.currency for cash in full.cash_end}
|
||
codes |= {position.currency for position in full.positions_end}
|
||
codes |= {event.currency for event in full.events}
|
||
codes |= {event.price_currency for event in trades(full)}
|
||
assert codes == {"RUB"}
|
||
|
||
|
||
def test_instrument_hint_reaches_trade_events(full: ParsedReport) -> None:
|
||
"""Only the positions table names the asset class («ПАЙ»); the trades table does not, so
|
||
the hint is carried over by ISIN."""
|
||
assert {ref.asset_class_hint for ref in full.instruments} == {"etf"}
|
||
assert all(
|
||
event.instrument is not None and event.instrument.asset_class_hint == "etf"
|
||
for event in trades(full)
|
||
)
|
||
|
||
|
||
# -- 5. the monthly report and the cross-file dedupe key ----------------------------------
|
||
|
||
|
||
def test_august_report(august: ParsedReport) -> None:
|
||
assert (august.period_from.isoformat(), august.period_to.isoformat()) == (
|
||
"2026-08-01",
|
||
"2026-08-31",
|
||
)
|
||
assert august.account_external_id == "77R593"
|
||
assert august.warnings == []
|
||
|
||
deals = trades(august)
|
||
assert len(deals) == 1
|
||
deal = deals[0]
|
||
assert deal.kind is EventKind.buy
|
||
assert deal.instrument is not None and deal.instrument.isin == "RU000A101EJ5"
|
||
assert deal.quantity == Decimal("33")
|
||
assert deal.price == Decimal("122.9")
|
||
assert deal.trade_no == AUGUST_TRADE_NO
|
||
# 33 * 122.9 = 4055.70 plus the 0.81 commission capitalised into the deal.
|
||
assert deal.amount == Decimal("-4056.51")
|
||
assert deal.fee == Decimal("0.81")
|
||
|
||
|
||
def test_same_deal_has_one_dedupe_key_in_both_reports(
|
||
full: ParsedReport, august: ParsedReport
|
||
) -> None:
|
||
"""The overlapping-period case from plan §1.6 A: re-importing August after the full
|
||
report must upsert, not duplicate."""
|
||
keys = {}
|
||
for report in (full, august):
|
||
for event in trades(report):
|
||
if event.trade_no == AUGUST_TRADE_NO:
|
||
keys[report.period_to] = common.dedupe_key(report.account_external_id, event)
|
||
assert len(keys) == 2
|
||
assert len(set(keys.values())) == 1
|
||
|
||
|
||
def test_dedupe_key_uses_the_brokers_own_deal_number(full: ParsedReport) -> None:
|
||
"""«№ сделки» (Z), not «№ сделки у организатора торгов» (AC).
|
||
|
||
The key is `sha1(broker | account | trade_no)` and is therefore already unique within the
|
||
broker; what it needs is the number VTB prints in every rendering of its own report. The
|
||
organiser's number is a different string (`17476399721` against `B17476399721`) and is
|
||
absent for over-the-counter deals in some formats.
|
||
"""
|
||
deal = next(event for event in trades(full) if event.trade_no == AUGUST_TRADE_NO)
|
||
assert deal.meta["exchange_trade_no"] == "17476399721"
|
||
assert common.dedupe_key("77R593", deal) != common.dedupe_key(
|
||
"77R593",
|
||
type(deal)(
|
||
**{
|
||
**{field.name: getattr(deal, field.name) for field in fields(deal)},
|
||
"trade_no": deal.meta["exchange_trade_no"],
|
||
}
|
||
),
|
||
)
|
||
|
||
|
||
def test_cash_rows_get_distinct_fingerprint_keys(full: ParsedReport) -> None:
|
||
"""Deposits carry no deal number and fall back to a fingerprint; two 4 000 ₽ top-ups in
|
||
different months must still differ."""
|
||
keys = [
|
||
common.dedupe_key(full.account_external_id, event)
|
||
for event in full.events
|
||
if event.kind is EventKind.deposit
|
||
]
|
||
assert len(set(keys)) == len(keys) == 8
|
||
|
||
|
||
# -- 6. concluded vs completed ------------------------------------------------------------
|
||
|
||
|
||
def test_concluded_and_completed_deal_numbers_agree(full: ParsedReport) -> None:
|
||
"""Asserted through the parser's own warning channel: a divergence is reported, and for a
|
||
fully settled period there is none."""
|
||
assert not [warning for warning in full.warnings if "Завершенных" in warning]
|
||
parsed = VtbXlsxParser().parse(FULL.read_bytes(), FULL.name)
|
||
concluded = {event.trade_no for event in trades(parsed)}
|
||
assert concluded == set(_completed_deal_numbers(FULL))
|
||
|
||
|
||
def _completed_deal_numbers(path: Path) -> list[str]:
|
||
"""«№ сделки» of «Завершенные…» read straight off the sheet, bypassing the parser."""
|
||
workbook = load_workbook(io.BytesIO(path.read_bytes()), read_only=True, data_only=True)
|
||
sheet = workbook["brokerage_report"]
|
||
sheet.reset_dimensions()
|
||
inside = False
|
||
numbers: list[str] = []
|
||
for row in sheet.iter_rows(values_only=True):
|
||
label = common.normalize_text(row[1] if len(row) > 1 else None)
|
||
if label.startswith("завершенные в отчетном периоде сделки с ценными бумагами"):
|
||
inside = True
|
||
continue
|
||
if not inside:
|
||
continue
|
||
value = row[25] if len(row) > 25 else None # column Z
|
||
if value and str(value).strip() and common.normalize_text(value) != "№ сделки":
|
||
numbers.append(str(value).strip())
|
||
workbook.close()
|
||
return numbers
|
||
|
||
|
||
# -- 7. no floats anywhere ----------------------------------------------------------------
|
||
|
||
|
||
def _floats(value: object, path: str = "") -> list[str]:
|
||
if isinstance(value, bool) or value is None:
|
||
return []
|
||
if isinstance(value, float):
|
||
return [path]
|
||
if is_dataclass(value) and not isinstance(value, type):
|
||
found: list[str] = []
|
||
for field in fields(value):
|
||
found += _floats(getattr(value, field.name), f"{path}.{field.name}")
|
||
return found
|
||
if isinstance(value, dict):
|
||
return [p for key, item in value.items() for p in _floats(item, f"{path}[{key}]")]
|
||
if isinstance(value, (list, tuple, set)):
|
||
return [p for index, item in enumerate(value) for p in _floats(item, f"{path}[{index}]")]
|
||
return []
|
||
|
||
|
||
@pytest.mark.parametrize("name", ["full", "august"])
|
||
def test_no_floats_in_output(name: str, request: pytest.FixtureRequest) -> None:
|
||
"""openpyxl hands back floats for every numeric cell; none of them may survive into the
|
||
result, or a kopeck disappears into binary rounding on the way to `NUMERIC(24,10)`."""
|
||
report: ParsedReport = request.getfixturevalue(name)
|
||
assert _floats(report, name) == []
|
||
|
||
|
||
# -- 8. totals against the summary block --------------------------------------------------
|
||
|
||
|
||
@pytest.mark.parametrize(("name", "expected"), [("full", "-25975.37"), ("august", "-4055.7")])
|
||
def test_trade_sums_match_the_settlement_saldo(
|
||
name: str, expected: str, request: pytest.FixtureRequest
|
||
) -> None:
|
||
"""Σ «Сумма сделки» (column M) of the purchases minus the sales equals «Сальдо расчетов по
|
||
сделкам с ценными бумагами» in the summary.
|
||
|
||
The commission has to be taken back out of `amount` to make the comparison: VTB's saldo is
|
||
the money of the deals themselves, while `event.amount` carries the fee inside it by the
|
||
ledger's convention. The two agree exactly once that is done — the fee lives in the same
|
||
file as its own «Вознаграждение брокера» line, and it is that line, not the saldo, that
|
||
accounts for it.
|
||
"""
|
||
report: ParsedReport = request.getfixturevalue(name)
|
||
gross = Decimal(0)
|
||
for event in trades(report):
|
||
fee = event.fee or Decimal(0)
|
||
gross += event.amount + fee if event.kind is EventKind.buy else event.amount - fee
|
||
assert gross == Decimal(expected)
|
||
assert gross == Decimal(report.meta["summary"]["сальдо расчетов по сделкам с ценными бумагами"])
|
||
|
||
|
||
def test_position_quantities_match_the_settled_movements(full: ParsedReport) -> None:
|
||
"""Closing position = what the trades did, since the account opened empty."""
|
||
by_isin: dict[str, Decimal] = {}
|
||
for event in trades(full):
|
||
assert event.instrument is not None and event.instrument.isin
|
||
assert event.quantity is not None
|
||
by_isin[event.instrument.isin] = (
|
||
by_isin.get(event.instrument.isin, Decimal(0)) + event.quantity
|
||
)
|
||
for position in full.positions_end:
|
||
assert position.instrument.isin is not None
|
||
assert by_isin[position.instrument.isin] == position.qty
|
||
|
||
|
||
# -- parse errors --------------------------------------------------------------------------
|
||
|
||
|
||
def test_unreadable_file_raises_parse_error(parser: VtbXlsxParser) -> None:
|
||
from fintracker.sources.reports.base import ParseError
|
||
|
||
with pytest.raises(ParseError):
|
||
parser.parse(b"PK\x03\x04 definitely not a workbook", "report.xlsx")
|
||
|
||
|
||
def test_workbook_without_a_period_raises_parse_error(parser: VtbXlsxParser) -> None:
|
||
"""A file we recognise but cannot place on a timeline must fail loudly rather than import
|
||
as an empty report."""
|
||
from openpyxl import Workbook
|
||
|
||
from fintracker.sources.reports.base import ParseError
|
||
|
||
workbook = Workbook()
|
||
sheet = workbook.worksheets[0]
|
||
sheet.title = "brokerage_report"
|
||
sheet["D1"] = "Отчет Банка ВТБ (ПАО) без периода"
|
||
buffer = io.BytesIO()
|
||
workbook.save(buffer)
|
||
|
||
with pytest.raises(ParseError):
|
||
parser.parse(buffer.getvalue(), "report.xlsx")
|