feat(reports): протокол и парсеры отчётов Сбера, ВТБ и Snowball CSV
sources/reports/base.py — контракт ReportParser (плагины: sniff/parse, чистые функции без БД). registry.py выбирает парсер по содержимому файла, CSV последним: он узнаёт файл по набору колонок и иначе перехватил бы чужой формат. Комиссия капитализируется в сделку, отдельным событием не эмитится: и Сбер, и ВТБ печатают её дважды — колонками в сделках и строками в движении денег, суммы совпадают, второе прочтение задвоило бы её. Расчётные строки («Сделка от …», «Сальдо расчетов по сделкам») не эмитятся — это денежные ноги уже учтённых сделок. У Сбера таблица «Информация о зачислениях на ИИС» кумулятивна за календарный год и в леджер не идёт, иначе два пересекающихся отчёта задвоили бы пополнения. CSV Snowball сводный по всем брокерам — годится как сверка потоков и сделок на уровне портфеля, но не позиций по счетам; CUSTOM_HOLDING_PRICE не событие, а цена — уходит в meta для price_manual. Обезличивание — anonymize.py + scripts/anonymize_reports.py, секреты собираются по всему корпусу отчётов разом (Snowball цитирует номер договора Сбера в примечании к переводу). test_fixtures_anonymized.py падает, если в tests/fixtures/reports/ вне raw/ найдётся ИНН, ФИО или номер счёта — и по форме (работает в CI без raw/), и по фактическому содержимому raw/, когда оно на месте.
This commit is contained in:
Dmitry
@@ -0,0 +1,420 @@
|
||||
"""The universal event-CSV parser, checked against the real Snowball export.
|
||||
|
||||
The fixture is the only file in the project that covers every brokerage account at once,
|
||||
so these tests double as its documentation: the exact number of rows of each type, which
|
||||
of them deliberately never become ledger events, and what the format's ambiguities were
|
||||
decided to mean. A change in any of those numbers is a change in the import contract, not
|
||||
a test that needs relaxing.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import collections
|
||||
import dataclasses
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import BrokerEvent, ParsedReport, fingerprint_key
|
||||
from fintracker.sources.reports.csv_universal import UniversalCsvParser
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports"
|
||||
SNOWBALL = FIXTURES / "snowball"
|
||||
|
||||
HEADER = (
|
||||
"Event,Date,Symbol,Price,Quantity,Currency,FeeTax,Exchange,NKD,FeeCurrency,DoNotAdjustCash,Note"
|
||||
)
|
||||
|
||||
#: What the fixture must yield, per kind. Sums to 535 of its 545 rows: 5 CUSTOM_HOLDING_PRICE
|
||||
#: and 1 CUSTOM_HOLDING_SETTINGS are not events at all, and 4 cash rows are Snowball's own
|
||||
#: balance corrections.
|
||||
EXPECTED_COUNTS = {
|
||||
EventKind.buy: 270,
|
||||
EventKind.sell: 60,
|
||||
EventKind.dividend: 80,
|
||||
EventKind.deposit: 96,
|
||||
EventKind.withdrawal: 5,
|
||||
EventKind.amortization: 9,
|
||||
EventKind.commission: 7,
|
||||
EventKind.tax: 4,
|
||||
EventKind.tax_refund: 2,
|
||||
EventKind.stock_split: 2,
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser() -> UniversalCsvParser:
|
||||
return UniversalCsvParser()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def fixture_path() -> Path:
|
||||
files = sorted(SNOWBALL.glob("*.csv"))
|
||||
assert files, f"нет фикстуры в {SNOWBALL}"
|
||||
return files[0]
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def report(parser: UniversalCsvParser, fixture_path: Path) -> ParsedReport:
|
||||
return parser.parse(fixture_path.read_bytes(), fixture_path.name)
|
||||
|
||||
|
||||
def _csv(*rows: str) -> bytes:
|
||||
return ("" + "\n".join((HEADER, *rows)) + "\n").encode("utf-8")
|
||||
|
||||
|
||||
def _key(report: ParsedReport, event: BrokerEvent) -> str:
|
||||
"""The dedupe key `ledger/ingest.py` will build for this event (plan §1.6 A)."""
|
||||
return fingerprint_key(
|
||||
report.broker,
|
||||
report.account_external_id,
|
||||
event.kind,
|
||||
event.instrument.key() if event.instrument else "",
|
||||
event.trade_date,
|
||||
event.quantity,
|
||||
event.price,
|
||||
event.currency,
|
||||
amount=event.amount,
|
||||
seq=event.seq,
|
||||
)
|
||||
|
||||
|
||||
def _floats(value: object, path: str) -> list[str]:
|
||||
"""Every float reachable from `value`, by path — money must be Decimal end to end."""
|
||||
if isinstance(value, bool | str | bytes) or value is None:
|
||||
return []
|
||||
if isinstance(value, float):
|
||||
return [f"{path} = {value!r}"]
|
||||
if dataclasses.is_dataclass(value) and not isinstance(value, type):
|
||||
return [
|
||||
bad
|
||||
for f in dataclasses.fields(value)
|
||||
for bad in _floats(getattr(value, f.name), f"{path}.{f.name}")
|
||||
]
|
||||
if isinstance(value, dict):
|
||||
return [bad for k, v in value.items() for bad in _floats(v, f"{path}[{k!r}]")]
|
||||
if isinstance(value, list | tuple | set):
|
||||
return [bad for i, v in enumerate(value) for bad in _floats(v, f"{path}[{i}]")]
|
||||
return []
|
||||
|
||||
|
||||
# --- 1. sniff -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sniff_recognises_the_fixture(parser: UniversalCsvParser, fixture_path: Path) -> None:
|
||||
assert parser.sniff(fixture_path.read_bytes(), fixture_path.name) is True
|
||||
|
||||
|
||||
def test_sniff_ignores_the_filename(parser: UniversalCsvParser) -> None:
|
||||
"""Recognition is by the header, so a renamed export is still recognised..."""
|
||||
data = _csv('"CASH_IN","2025-02-26 09:22:54","RUB","1","5000","RUB","0","","","","False",""')
|
||||
assert parser.sniff(data, "какой-то-файл.csv") is True
|
||||
assert parser.sniff(data, "") is True
|
||||
|
||||
|
||||
def test_sniff_rejects_other_report_formats(parser: UniversalCsvParser) -> None:
|
||||
"""...and a file that merely has an extension we read is not ours."""
|
||||
others = sorted((FIXTURES / "sber").glob("*.html")) + sorted((FIXTURES / "vtb").glob("*.xlsx"))
|
||||
assert others, "нет чужих фикстур для отрицательной проверки"
|
||||
for path in others:
|
||||
assert parser.sniff(path.read_bytes(), path.name) is False, path.name
|
||||
|
||||
|
||||
def test_sniff_rejects_a_csv_with_foreign_columns(parser: UniversalCsvParser) -> None:
|
||||
assert parser.sniff(b"a,b,c\n1,2,3\n", "x.csv") is False
|
||||
assert parser.sniff(b"", "x.csv") is False
|
||||
assert parser.sniff(b"\x00\x01\x02", "x.csv") is False
|
||||
|
||||
|
||||
def test_parse_rejects_a_file_it_cannot_read(parser: UniversalCsvParser) -> None:
|
||||
from fintracker.sources.reports.base import ParseError
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse(b"a,b,c\n1,2,3\n", "x.csv")
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse(_csv(), "empty.csv")
|
||||
|
||||
|
||||
# --- 2. what the fixture yields --------------------------------------------------------
|
||||
|
||||
|
||||
def test_report_header(report: ParsedReport) -> None:
|
||||
import datetime
|
||||
|
||||
assert report.broker == "csv"
|
||||
assert report.parser_version == "1"
|
||||
# the file names a portfolio, never an account — see the warning below
|
||||
assert report.account_external_id == "Мой капитал"
|
||||
assert report.period_from == datetime.date(2025, 2, 26)
|
||||
assert report.period_to == datetime.date(2026, 9, 17)
|
||||
|
||||
|
||||
def test_event_counts_by_kind(report: ParsedReport) -> None:
|
||||
counts = collections.Counter(e.kind for e in report.events)
|
||||
assert dict(counts) == EXPECTED_COUNTS
|
||||
assert len(report.events) == 535
|
||||
assert report.meta["row_count"] == 545
|
||||
|
||||
|
||||
def test_instruments_are_deduplicated(report: ParsedReport) -> None:
|
||||
keys = [ref.key() for ref in report.instruments]
|
||||
assert len(keys) == len(set(keys)) == 49
|
||||
# bonds arrive by ISIN, everything else by ticker; cash rows name no instrument
|
||||
assert "ISIN:RU000A10B7T7" in keys
|
||||
assert "TICKER:SBER" in keys
|
||||
assert "TICKER:SU26212RMFS9" in keys, "секунда ОФЗ — не ISIN, а тикер MOEX"
|
||||
assert not any(k.startswith("TICKER:RUB") for k in keys)
|
||||
|
||||
|
||||
# --- 3. decimals ----------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_decimal_comma_is_parsed(parser: UniversalCsvParser) -> None:
|
||||
data = _csv(
|
||||
'"BUY","2025-03-01 10:00:00","SBER","4,48","219","RUB","6,57","MCX","0","RUB","False",""'
|
||||
)
|
||||
(event,) = parser.parse(data, "renamed.csv").events
|
||||
assert event.price == Decimal("4.48")
|
||||
assert event.quantity == Decimal("219")
|
||||
assert event.fee == Decimal("6.57")
|
||||
assert event.amount == Decimal("-987.69") # 4.48 * 219 + 6.57, fee capitalised
|
||||
|
||||
|
||||
def test_no_floats_anywhere_in_the_output(report: ParsedReport) -> None:
|
||||
bad = _floats(report, "report")
|
||||
assert bad == []
|
||||
|
||||
|
||||
def test_empty_numeric_cell_is_none_not_zero(parser: UniversalCsvParser) -> None:
|
||||
"""An absent НКД is not a zero НКД: one is «not a bond», the other is «no interest»."""
|
||||
data = _csv('"BUY","2025-03-01 10:00:00","SBER","100","1","RUB","0","MCX","","","False",""')
|
||||
(event,) = parser.parse(data, "x.csv").events
|
||||
assert event.accrued_interest is None
|
||||
assert event.fee == Decimal(0)
|
||||
|
||||
|
||||
def test_bond_trade_adds_accrued_interest_to_the_money(report: ParsedReport) -> None:
|
||||
"""НКД is paid on top of the price by the buyer — it must be inside `amount`."""
|
||||
buy = next(
|
||||
e
|
||||
for e in report.events
|
||||
if e.kind is EventKind.buy
|
||||
and e.instrument is not None
|
||||
and e.instrument.isin == "RU000A107AM4"
|
||||
and e.accrued_interest == Decimal("5.08")
|
||||
)
|
||||
assert buy.quantity == Decimal(4)
|
||||
assert buy.price == Decimal("929.25")
|
||||
assert buy.amount == -(Decimal(4) * Decimal("929.25") + Decimal("5.08") + Decimal("11.15"))
|
||||
|
||||
|
||||
# --- 4. dividends and coupons ----------------------------------------------------------
|
||||
|
||||
|
||||
def test_dividend_money_lands_in_amount_not_quantity(report: ParsedReport) -> None:
|
||||
dividends = [e for e in report.events if e.kind is EventKind.dividend]
|
||||
assert len(dividends) == 80
|
||||
assert all(e.quantity is None for e in dividends), "Quantity у DIVIDEND — это деньги"
|
||||
assert all(e.price is None for e in dividends)
|
||||
assert all(e.amount > 0 for e in dividends)
|
||||
assert sum(e.amount for e in dividends) == Decimal("12949.07")
|
||||
|
||||
|
||||
def test_bond_dividends_carry_the_coupon_hint(report: ParsedReport) -> None:
|
||||
"""The parser cannot know an asset class, so it hands ingest a suspicion, not a kind."""
|
||||
dividends = [e for e in report.events if e.kind is EventKind.dividend]
|
||||
hinted = [e for e in dividends if e.meta.get("payout_hint") == "coupon"]
|
||||
assert len(hinted) == 44
|
||||
assert all(
|
||||
e.instrument is not None
|
||||
and (
|
||||
(e.instrument.isin or "").startswith("RU000A")
|
||||
or (e.instrument.ticker or "").startswith("SU")
|
||||
)
|
||||
for e in hinted
|
||||
)
|
||||
# a share payout must never be hinted
|
||||
assert all(
|
||||
e.meta.get("payout_hint") is None
|
||||
for e in dividends
|
||||
if e.instrument is not None and e.instrument.ticker in {"T", "ROSN", "SBERP", "TATNP"}
|
||||
)
|
||||
|
||||
|
||||
# --- 5. splits -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_split_ratio_and_collapsing_dedupe_key(report: ParsedReport) -> None:
|
||||
"""Two identical SPLIT rows for T are one corporate action seen on two accounts.
|
||||
|
||||
The format has no account column, so both rows land on the single account the user
|
||||
points the import at. They therefore share a `fingerprint_key` on purpose: replaying the
|
||||
ratio twice would multiply the position by 100 instead of by 10. This test pins that
|
||||
behaviour — if a future format gains an account column, it is expected to break.
|
||||
"""
|
||||
splits = [e for e in report.events if e.kind is EventKind.stock_split]
|
||||
assert len(splits) == 2
|
||||
assert {e.meta["split_ratio"] for e in splits} == {Decimal(10)}
|
||||
assert {e.meta["ratio"] for e in splits} == {Decimal(10)}
|
||||
assert all(e.amount == Decimal(0) and e.quantity is None for e in splits)
|
||||
assert all(e.instrument is not None and e.instrument.ticker == "T" for e in splits)
|
||||
assert [e.seq for e in splits] == [0, 0]
|
||||
assert _key(report, splits[0]) == _key(report, splits[1])
|
||||
|
||||
|
||||
def test_every_other_event_keeps_a_distinct_dedupe_key(report: ParsedReport) -> None:
|
||||
"""The split pair is the ONLY intentional collapse in the file.
|
||||
|
||||
In particular the pairs of equal same-day top-ups (two 1 100 ₽ one second apart, two
|
||||
4 000 ₽ that differ only by their note) survive as distinct events: they are real money
|
||||
into two accounts, and netting them would understate the XIRR denominator.
|
||||
"""
|
||||
keys = [_key(report, e) for e in report.events]
|
||||
collapsed = [k for k, n in collections.Counter(keys).items() if n > 1]
|
||||
assert len(collapsed) == 1
|
||||
assert len(set(keys)) == 534
|
||||
same_second = [e for e in report.events if e.kind is EventKind.deposit and e.seq > 0]
|
||||
assert same_second, "в файле есть одинаковые пополнения одного дня — они должны выживать"
|
||||
|
||||
|
||||
def test_seq_is_stable_across_reparsing(parser: UniversalCsvParser, fixture_path: Path) -> None:
|
||||
"""A re-import of the same bytes must reproduce every key, or nothing ever upserts."""
|
||||
data = fixture_path.read_bytes()
|
||||
first = parser.parse(data, fixture_path.name)
|
||||
second = parser.parse(data, "совсем другое имя.csv")
|
||||
assert [_key(first, e) for e in first.events] == [_key(first, e) for e in second.events]
|
||||
|
||||
|
||||
# --- 6. amortisation -------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_amortisation_is_money_on_a_bond(report: ParsedReport) -> None:
|
||||
amortisations = [e for e in report.events if e.kind is EventKind.amortization]
|
||||
assert len(amortisations) == 9
|
||||
assert all(e.amount > 0 for e in amortisations)
|
||||
assert all(e.quantity is None and e.price is None for e in amortisations)
|
||||
assert {e.instrument.isin for e in amortisations if e.instrument} == {"RU000A10B7T7"}
|
||||
assert all(
|
||||
e.instrument is not None and e.instrument.asset_class_hint == "bond" for e in amortisations
|
||||
)
|
||||
assert sum(e.amount for e in amortisations) == Decimal("4530.07")
|
||||
|
||||
|
||||
# --- 7. manual prices ------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_custom_holding_prices_go_to_meta_not_to_events(report: ParsedReport) -> None:
|
||||
"""CUSTOM_HOLDING_PRICE is a `price_manual` row, not a ledger event."""
|
||||
prices = report.meta["manual_prices"]
|
||||
assert len(prices) == 5
|
||||
assert {p["instrument_key"] for p in prices} == {"TICKER:SIBN6P4"}
|
||||
assert {p["currency"] for p in prices} == {"RUB"}
|
||||
assert all(isinstance(p["price"], Decimal) for p in prices)
|
||||
assert max(p["price"] for p in prices) == Decimal("12320.94504")
|
||||
assert all(e.meta.get("csv_event") != "CUSTOM_HOLDING_PRICE" for e in report.events)
|
||||
# the instrument they price must be resolvable, so it is also listed
|
||||
assert "TICKER:SIBN6P4" in {ref.key() for ref in report.instruments}
|
||||
|
||||
|
||||
def test_the_export_has_no_price_for_the_other_unquoted_paper(report: ParsedReport) -> None:
|
||||
"""NDM_TBNK-PP-FIXPRCNT-08.25 is simply absent — this file cannot price it."""
|
||||
named = {(ref.ticker or "") + (ref.isin or "") for ref in report.instruments}
|
||||
assert not any("NDM" in name for name in named)
|
||||
assert not any("NDM" in p["instrument_key"] for p in report.meta["manual_prices"])
|
||||
|
||||
|
||||
def test_custom_holding_settings_describe_an_instrument(report: ParsedReport) -> None:
|
||||
"""Its Note is JSON with `"` rewritten to `@*@` and Cyrillic as \\uXXXX escapes."""
|
||||
ref = next(r for r in report.instruments if r.ticker == "SIBN6P4")
|
||||
assert ref.name == "Газпром Нефть 006Р-04"
|
||||
assert ref.currency == "RUB"
|
||||
assert ref.meta["custom_holding"] is True
|
||||
assert ref.meta["sector"] == "Other"
|
||||
assert ref.meta["settings"]["CustomHoldingType"] == 0
|
||||
assert all(e.meta.get("csv_event") != "CUSTOM_HOLDING_SETTINGS" for e in report.events)
|
||||
|
||||
|
||||
def test_broken_settings_json_is_a_warning_not_a_crash(parser: UniversalCsvParser) -> None:
|
||||
data = _csv(
|
||||
'"CUSTOM_HOLDING_SETTINGS","2026-09-17 00:00:00","XYZ","0","0","RUB","0",'
|
||||
'"CUSTOM_HOLDING","0","","","{@*@Holding@*@:'
|
||||
)
|
||||
result = parser.parse(data, "x.csv")
|
||||
assert result.events == []
|
||||
assert any("не удалось разобрать JSON" in w for w in result.warnings)
|
||||
assert [ref.key() for ref in result.instruments] == ["TICKER:XYZ"]
|
||||
|
||||
|
||||
# --- 8. balance adjustments ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_balance_adjustments_never_become_flows(report: ParsedReport) -> None:
|
||||
"""Snowball's own corrections would feed XIRR an external flow that never happened."""
|
||||
adjustments = report.meta["balance_adjustments"]
|
||||
assert len(adjustments) == 4
|
||||
assert collections.Counter(a["kind"] for a in adjustments) == {"CASH_OUT": 3, "CASH_IN": 1}
|
||||
assert all(isinstance(a["amount"], Decimal) for a in adjustments)
|
||||
# detection is by the note, not by the amount: one correction is 1017,84 ₽ while a
|
||||
# genuine withdrawal in the same file is 32 ₽
|
||||
assert min(abs(a["amount"]) for a in adjustments) == Decimal("0.0024")
|
||||
assert max(abs(a["amount"]) for a in adjustments) == Decimal("1017.83999543")
|
||||
excluded = {(a["kind"], a["d"], abs(a["amount"])) for a in adjustments}
|
||||
for event in report.events:
|
||||
kind = "CASH_IN" if event.kind is EventKind.deposit else "CASH_OUT"
|
||||
assert (kind, event.trade_date, abs(event.amount)) not in excluded
|
||||
assert any("правка остатка" in w for w in report.warnings)
|
||||
|
||||
|
||||
def test_real_cash_flows_survive(report: ParsedReport) -> None:
|
||||
deposits = [e for e in report.events if e.kind is EventKind.deposit]
|
||||
withdrawals = [e for e in report.events if e.kind is EventKind.withdrawal]
|
||||
assert len(deposits) == 96 and len(withdrawals) == 5
|
||||
assert all(e.amount > 0 and e.instrument is None for e in deposits)
|
||||
assert all(e.amount < 0 and e.instrument is None for e in withdrawals)
|
||||
assert sum(e.amount for e in withdrawals) == -Decimal("51294.41")
|
||||
assert sum(e.amount for e in deposits) == Decimal("454058.45")
|
||||
|
||||
|
||||
# --- 9. fees, taxes, refunds -----------------------------------------------------------
|
||||
|
||||
|
||||
def test_fee_tax_and_refund_signs(report: ParsedReport) -> None:
|
||||
fees = [e for e in report.events if e.kind is EventKind.commission]
|
||||
taxes = [e for e in report.events if e.kind is EventKind.tax]
|
||||
refunds = [e for e in report.events if e.kind is EventKind.tax_refund]
|
||||
assert (len(fees), len(taxes), len(refunds)) == (7, 4, 2)
|
||||
assert all(e.instrument is None for e in fees + taxes + refunds)
|
||||
assert all(e.quantity is None and e.price is None for e in fees + taxes + refunds)
|
||||
assert all(e.amount == Decimal(-45) and e.fee == Decimal(45) for e in fees)
|
||||
assert [e.amount for e in taxes] == [Decimal(-234), Decimal(-3), Decimal(-22), Decimal(-32)]
|
||||
assert [e.amount for e in refunds] == [Decimal(3), Decimal(32)]
|
||||
assert all(e.tax == -e.amount for e in taxes)
|
||||
assert all(e.tax == e.amount for e in refunds)
|
||||
|
||||
|
||||
# --- unknown types and reconciliation warnings -----------------------------------------
|
||||
|
||||
|
||||
def test_unknown_event_type_is_skipped_with_a_warning(parser: UniversalCsvParser) -> None:
|
||||
"""Never `EventKind.other`: a silent bucket hides a format that grew a new row type."""
|
||||
data = _csv(
|
||||
'"MARGIN_CALL","2025-03-01 10:00:00","SBER","1","1","RUB","0","MCX","0","","False",""',
|
||||
'"BUY","2025-03-01 10:00:00","SBER","100","1","RUB","0","MCX","0","","False",""',
|
||||
)
|
||||
result = parser.parse(data, "x.csv")
|
||||
assert [e.kind for e in result.events] == [EventKind.buy]
|
||||
assert result.meta["unknown_events"] == {"MARGIN_CALL": 1}
|
||||
assert any("MARGIN_CALL" in w for w in result.warnings)
|
||||
assert not any(e.kind is EventKind.other for e in result.events)
|
||||
|
||||
|
||||
def test_closing_balances_are_absent_and_said_to_be(report: ParsedReport) -> None:
|
||||
assert report.positions_end == []
|
||||
assert report.cash_end == []
|
||||
assert any("positions_end" in w for w in report.warnings)
|
||||
|
||||
|
||||
def test_warns_that_the_target_account_is_the_user_s_to_choose(report: ParsedReport) -> None:
|
||||
assert any("shadow" in w and "все брокерские счета" in w for w in report.warnings)
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Реестр парсеров отчётов: какой файл достаётся какому парсеру (plan §2).
|
||||
|
||||
Проверка выглядит тривиальной, но ловит ровно ту ошибку, которую больше не поймает никто:
|
||||
универсальный CSV узнаёт файл по набору колонок, то есть по определению шире любого
|
||||
брокерского формата, и достаточно поставить его в списке первым, чтобы отчёт Сбера тихо
|
||||
уехал в чужой парсер. Порядок в `PARSERS` — это поведение, а не оформление.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fintracker.sources.reports import registry
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports"
|
||||
|
||||
CASES = [
|
||||
(FIXTURES / "sber" / "S930W42_11022026_17092026.html", "report_sber"),
|
||||
(FIXTURES / "sber" / "S930W42_01082026_31082026.html", "report_sber"),
|
||||
(FIXTURES / "vtb" / "reportd0235f1e-55ac-fdc8-5c7b-279e4fe26e2b.xlsx", "report_vtb"),
|
||||
(FIXTURES / "vtb" / "report8ef8347a-2deb-e880-959a-d9a8ca12f735.xlsx", "report_vtb"),
|
||||
(next((FIXTURES / "snowball").glob("*.csv")), "csv"),
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("path,expected", CASES, ids=lambda v: getattr(v, "name", v))
|
||||
def test_pick_routes_each_fixture_to_its_parser(path: Path, expected: str) -> None:
|
||||
chosen = registry.pick(path.read_bytes(), path.name)
|
||||
assert chosen is not None, f"{path.name}: ни один парсер не узнал файл"
|
||||
assert chosen.name == expected
|
||||
|
||||
|
||||
def test_every_registered_parser_is_reachable_by_name() -> None:
|
||||
assert set(registry.names()) == {"report_sber", "report_vtb", "csv"}
|
||||
for name in registry.names():
|
||||
assert registry.get(name).name == name
|
||||
|
||||
|
||||
def test_unknown_format_is_a_none_not_an_exception() -> None:
|
||||
"""`pick` возвращает None, и роутер превращает это в 415 — не в 500."""
|
||||
assert registry.pick(b"%PDF-1.4\n%\xe2\xe3\xcf\xd3\n", "statement.pdf") is None
|
||||
assert registry.pick(b"", "empty.txt") is None
|
||||
|
||||
|
||||
def test_parser_metadata_is_filled_in() -> None:
|
||||
"""`parser_name` и `parser_version` уходят в `raw_report_file` — пустых быть не должно."""
|
||||
for parser in registry.PARSERS:
|
||||
assert parser.name and parser.broker and parser.version
|
||||
assert parser.formats
|
||||
|
||||
|
||||
def test_csv_is_last_so_it_cannot_shadow_a_broker_format() -> None:
|
||||
names = registry.names()
|
||||
assert names.index("csv") == len(names) - 1
|
||||
@@ -0,0 +1,362 @@
|
||||
"""Парсер отчётов Сбера на обезличенных фикстурах (фаза 3).
|
||||
|
||||
Фикстуры — два отчёта по одному и тому же счёту: полный (11.02–17.09.2026) и августовский
|
||||
(01.08–31.08.2026). Пара выбрана не случайно: именно перекрывающиеся периоды ломают импорт,
|
||||
если ключ дедупликации выводится из файла, а не из самой операции.
|
||||
|
||||
Главная проверка здесь — арифметическая. Отчёт печатает свои итоги («Итого, RUB» по
|
||||
сделкам, «Пополнение счета» и «Исходящий остаток» в сводке), и распарсенные события обязаны
|
||||
их воспроизвести: пополнения минус покупки плюс продажи = остаток на конец периода. Если
|
||||
парсер потеряет строку, задвоит комиссию или примет расчётную строку за сделку, это
|
||||
равенство разойдётся — а тест на «количество событий» этого не заметит.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections import Counter
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import BrokerEvent, ParsedReport
|
||||
from fintracker.sources.reports.sber import SberHtmlParser, event_dedupe_key
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports"
|
||||
FULL = FIXTURES / "sber" / "S930W42_11022026_17092026.html"
|
||||
AUGUST = FIXTURES / "sber" / "S930W42_01082026_31082026.html"
|
||||
|
||||
ACCOUNT = "S930W42"
|
||||
#: Второй договор того же брокера — источник переводов д/с, не внешний поток.
|
||||
OTHER_AGREEMENT = "8184V30"
|
||||
|
||||
ZERO = Decimal(0)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser() -> SberHtmlParser:
|
||||
return SberHtmlParser()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def full(parser: SberHtmlParser) -> ParsedReport:
|
||||
return parser.parse(FULL.read_bytes(), FULL.name)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def august(parser: SberHtmlParser) -> ParsedReport:
|
||||
return parser.parse(AUGUST.read_bytes(), AUGUST.name)
|
||||
|
||||
|
||||
def total(events: list[BrokerEvent], kind: EventKind) -> Decimal:
|
||||
return sum((e.amount for e in events if e.kind == kind), ZERO)
|
||||
|
||||
|
||||
def of_kind(report: ParsedReport, kind: EventKind) -> list[BrokerEvent]:
|
||||
return [e for e in report.events if e.kind == kind]
|
||||
|
||||
|
||||
# --- 1. распознавание формата ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sniff_accepts_both_sber_reports(parser: SberHtmlParser) -> None:
|
||||
assert parser.sniff(FULL.read_bytes(), FULL.name)
|
||||
assert parser.sniff(AUGUST.read_bytes(), AUGUST.name)
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"path",
|
||||
sorted((FIXTURES / "vtb").glob("*.xlsx")) + sorted((FIXTURES / "snowball").glob("*.csv")),
|
||||
ids=lambda p: p.suffix,
|
||||
)
|
||||
def test_sniff_rejects_other_formats(parser: SberHtmlParser, path: Path) -> None:
|
||||
assert not parser.sniff(path.read_bytes(), path.name)
|
||||
|
||||
|
||||
def test_sniff_never_raises_on_garbage(parser: SberHtmlParser) -> None:
|
||||
"""`registry.pick` опрашивает парсеры подряд — падение на чужом файле остановило бы перебор."""
|
||||
assert not parser.sniff(b"", "empty.html")
|
||||
assert not parser.sniff(b"\x00\x01\x02not html at all", "junk.html")
|
||||
|
||||
|
||||
# --- 2. шапка и состав ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_report_header(full: ParsedReport) -> None:
|
||||
assert full.broker == "sber"
|
||||
assert full.account_external_id == ACCOUNT
|
||||
assert (full.period_from.isoformat(), full.period_to.isoformat()) == (
|
||||
"2026-02-11",
|
||||
"2026-09-17",
|
||||
)
|
||||
assert full.meta["opened_at"] == "2026-02-11"
|
||||
|
||||
|
||||
def test_august_report_header(august: ParsedReport) -> None:
|
||||
assert august.account_external_id == ACCOUNT
|
||||
assert (august.period_from.isoformat(), august.period_to.isoformat()) == (
|
||||
"2026-08-01",
|
||||
"2026-08-31",
|
||||
)
|
||||
|
||||
|
||||
def test_full_report_event_counts(full: ParsedReport) -> None:
|
||||
assert Counter(e.kind for e in full.events) == {
|
||||
EventKind.buy: 24,
|
||||
EventKind.deposit: 9,
|
||||
EventKind.sell: 2,
|
||||
}
|
||||
|
||||
|
||||
def test_august_report_event_counts(august: ParsedReport) -> None:
|
||||
assert Counter(e.kind for e in august.events) == {
|
||||
EventKind.buy: 2,
|
||||
EventKind.deposit: 1,
|
||||
}
|
||||
|
||||
|
||||
# --- 3. арифметика: события воспроизводят итоги самого отчёта -------------------------------
|
||||
|
||||
|
||||
def test_trade_totals_match_the_reports_own_total_row(full: ParsedReport) -> None:
|
||||
"""«Итого, RUB» таблицы сделок: 89 205.58 оборота, 138.03 брокеру, 19.54 бирже."""
|
||||
trades = of_kind(full, EventKind.buy) + of_kind(full, EventKind.sell)
|
||||
assert len(trades) == 26
|
||||
|
||||
# Оборот — это «Сумма» сделки без комиссии, а `amount` её уже включает: у покупки
|
||||
# прибавляет, у продажи вычитает. Отсюда знаки при обратном пересчёте.
|
||||
turnover = sum(
|
||||
(-e.amount - (e.fee or ZERO) if e.kind == EventKind.buy else e.amount + (e.fee or ZERO))
|
||||
for e in trades
|
||||
)
|
||||
assert turnover == Decimal("89205.58")
|
||||
assert sum((e.fee or ZERO for e in full.events), ZERO) == Decimal("138.03") + Decimal("19.54")
|
||||
|
||||
|
||||
def test_deposits_match_the_summary_line(full: ParsedReport) -> None:
|
||||
"""«Пополнение счета» сводки считает и переводы с другого договора — как и парсер."""
|
||||
assert total(full.events, EventKind.deposit) == Decimal("49120.87")
|
||||
assert Decimal(full.meta["summary"]["Пополнение счета"]) == Decimal("49120.87")
|
||||
|
||||
|
||||
def test_cash_closes_against_the_reported_balance(full: ParsedReport) -> None:
|
||||
"""Сквозная проверка: сумма денежных эффектов всех событий = исходящий остаток.
|
||||
|
||||
Это единственная проверка, которая ловит и потерянную строку, и лишнюю: пропущенная
|
||||
сделка занижает отток, а расчётная строка «Сделка от …», принятая за сделку, задваивает
|
||||
его. Входящий остаток нулевой, поэтому сумма событий и есть конечный остаток.
|
||||
"""
|
||||
assert Decimal(full.meta["summary"]["Входящий остаток"]) == ZERO
|
||||
assert sum((e.amount for e in full.events), ZERO) == Decimal("3171.34")
|
||||
|
||||
rub = next(c for c in full.cash_end if c.currency == "RUB")
|
||||
assert rub.balance == Decimal("3171.34")
|
||||
|
||||
|
||||
def test_august_cash_closes_from_its_own_opening_balance(august: ParsedReport) -> None:
|
||||
opening = Decimal(august.meta["summary"]["Входящий остаток"])
|
||||
assert opening == Decimal("357.15")
|
||||
rub = next(c for c in august.cash_end if c.currency == "RUB")
|
||||
assert opening + sum((e.amount for e in august.events), ZERO) == rub.balance
|
||||
|
||||
|
||||
# --- 4. комиссии учтены ровно один раз -----------------------------------------------------
|
||||
|
||||
|
||||
def test_commission_is_capitalised_and_never_emitted_separately(full: ParsedReport) -> None:
|
||||
"""Комиссия живёт в `fee` сделки; отдельных `commission`-событий быть не должно.
|
||||
|
||||
`ledger/lots.py` капитализирует `fee` в стоимость лота, поэтому вторая, «денежная»
|
||||
ипостась той же комиссии (строки «Комиссия Брокера от …» в движении денег) ушла бы в
|
||||
леджер повторно — как отток, которого не было.
|
||||
"""
|
||||
assert not of_kind(full, EventKind.commission)
|
||||
assert all(e.fee is None for e in full.events if e.kind == EventKind.deposit)
|
||||
|
||||
fees = sum((e.fee or ZERO for e in full.events), ZERO)
|
||||
assert fees == Decimal("157.57")
|
||||
assert Decimal(full.meta["summary"]["Комиссия брокера"]) == Decimal("-138.03")
|
||||
assert Decimal(full.meta["summary"]["Комиссия биржи"]) == Decimal("-19.54")
|
||||
|
||||
|
||||
def test_parser_reports_no_fee_divergence(full: ParsedReport, august: ParsedReport) -> None:
|
||||
"""Парсер сам сверяет две ипостаси комиссии и предупреждает при расхождении."""
|
||||
for report in (full, august):
|
||||
assert not [w for w in report.warnings if "комиссии расходятся" in w]
|
||||
|
||||
|
||||
# --- 5. дедупликация перекрывающихся периодов ----------------------------------------------
|
||||
|
||||
|
||||
def keys(report: ParsedReport) -> dict[str, BrokerEvent]:
|
||||
return {event_dedupe_key(report, e): e for e in report.events}
|
||||
|
||||
|
||||
def test_august_events_keep_their_keys_in_the_full_report(
|
||||
full: ParsedReport, august: ParsedReport
|
||||
) -> None:
|
||||
"""Каждое августовское событие должно прийти с тем же ключом из полного отчёта.
|
||||
|
||||
Иначе импорт двух пересекающихся отчётов задвоит август — ровно тот случай, который
|
||||
план §1.6 A называет «перекрывающиеся периоды апсертят в ту же строку».
|
||||
"""
|
||||
full_keys, august_keys = keys(full), keys(august)
|
||||
assert set(august_keys) <= set(full_keys), (
|
||||
"у августовских событий появились ключи, которых нет в полном отчёте: "
|
||||
f"{sorted(set(august_keys) - set(full_keys))}"
|
||||
)
|
||||
|
||||
for key, event in august_keys.items():
|
||||
twin = full_keys[key]
|
||||
assert (event.kind, event.trade_date, event.quantity, event.price, event.amount) == (
|
||||
twin.kind,
|
||||
twin.trade_date,
|
||||
twin.quantity,
|
||||
twin.price,
|
||||
twin.amount,
|
||||
), f"ключ {key} указывает на разные операции"
|
||||
|
||||
|
||||
def test_overlap_covers_every_august_operation(august: ParsedReport) -> None:
|
||||
"""Перекрытие не должно оказаться пустым — иначе предыдущий тест проходит вхолостую."""
|
||||
assert len(keys(august)) == len(august.events) == 3
|
||||
|
||||
|
||||
def test_trades_key_on_the_brokers_deal_number(full: ParsedReport) -> None:
|
||||
trades = of_kind(full, EventKind.buy) + of_kind(full, EventKind.sell)
|
||||
assert all(e.trade_no for e in trades)
|
||||
assert len({e.trade_no for e in trades}) == len(trades)
|
||||
|
||||
|
||||
def test_cash_rows_have_stable_keys_without_a_deal_number(full: ParsedReport) -> None:
|
||||
"""У денежных строк номера нет, и ключ обязан оставаться уникальным внутри отчёта."""
|
||||
deposits = of_kind(full, EventKind.deposit)
|
||||
assert all(e.trade_no is None for e in deposits)
|
||||
assert len({event_dedupe_key(full, e) for e in deposits}) == len(deposits)
|
||||
|
||||
|
||||
def test_reparsing_the_same_bytes_is_deterministic(parser: SberHtmlParser) -> None:
|
||||
once = parser.parse(FULL.read_bytes(), FULL.name)
|
||||
twice = parser.parse(FULL.read_bytes(), FULL.name)
|
||||
assert [event_dedupe_key(once, e) for e in once.events] == [
|
||||
event_dedupe_key(twice, e) for e in twice.events
|
||||
]
|
||||
|
||||
|
||||
# --- 6. закрывающие позиции и остатки ------------------------------------------------------
|
||||
|
||||
|
||||
def test_closing_positions(full: ParsedReport) -> None:
|
||||
assert len(full.positions_end) == 10
|
||||
assert sum((p.market_value or ZERO for p in full.positions_end), ZERO) == Decimal("46663.30")
|
||||
|
||||
aeroflot = next(p for p in full.positions_end if p.instrument.isin == "RU0009062285")
|
||||
assert aeroflot.qty == Decimal("130")
|
||||
assert aeroflot.currency == "RUB"
|
||||
|
||||
|
||||
def test_closing_cash_keeps_every_currency(full: ParsedReport) -> None:
|
||||
balances = {c.currency: c.balance for c in full.cash_end}
|
||||
assert balances == {"RUB": Decimal("3171.34"), "EUR": ZERO, "USD": ZERO}
|
||||
|
||||
|
||||
def test_august_closing_snapshot(august: ParsedReport) -> None:
|
||||
assert len(august.positions_end) == 5
|
||||
assert sum((p.market_value or ZERO for p in august.positions_end), ZERO) == Decimal("38739.43")
|
||||
assert next(c for c in august.cash_end if c.currency == "RUB").balance == Decimal("1407.88")
|
||||
|
||||
|
||||
# --- 7. справочник инструментов ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_securities_directory_feeds_instruments(full: ParsedReport) -> None:
|
||||
assert len(full.instruments) >= 9
|
||||
assert all(i.isin and i.ticker for i in full.instruments)
|
||||
assert all(i.asset_class_hint for i in full.instruments)
|
||||
|
||||
sber = next(i for i in full.instruments if i.ticker == "SBER")
|
||||
assert (sber.isin, sber.asset_class_hint) == ("RU0009029540", "share")
|
||||
fund = next(i for i in full.instruments if i.ticker == "STME")
|
||||
assert fund.asset_class_hint in {"etf", "fund"}
|
||||
|
||||
|
||||
def test_trades_reference_instruments_by_isin(full: ParsedReport) -> None:
|
||||
"""Сделка печатает только тикер — ISIN подставляется из справочника того же файла."""
|
||||
trades = of_kind(full, EventKind.buy) + of_kind(full, EventKind.sell)
|
||||
assert all(e.instrument is not None and e.instrument.isin for e in trades)
|
||||
|
||||
|
||||
# --- 8. ловушки формата --------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_iis_contributions_table_is_read_but_not_emitted(
|
||||
full: ParsedReport, august: ParsedReport
|
||||
) -> None:
|
||||
"""Таблица зачислений на ИИС кумулятивна за ГОД, а не за период отчёта.
|
||||
|
||||
Августовский файл перечисляет в ней восемь пополнений начиная с февраля — всё, что
|
||||
случилось до даты его формирования (01.09.2026); в полном отчёте их девять, добавилось
|
||||
сентябрьское. То есть таблица растёт от отчёта к отчёту независимо от периода, и если
|
||||
бы парсер эмитил её строки, импорт августовского файла поверх полного задвоил бы
|
||||
пополнения за полгода. Поэтому её содержимое только пересчитывается, в события не идёт:
|
||||
августовский отчёт даёт ровно одно денежное поступление — то, что реально было в августе.
|
||||
"""
|
||||
assert full.meta["iis_contributions_ignored"] == 9
|
||||
assert august.meta["iis_contributions_ignored"] == 8
|
||||
|
||||
assert len(of_kind(august, EventKind.deposit)) == 1
|
||||
|
||||
|
||||
def test_settlement_lines_are_not_mistaken_for_trades(full: ParsedReport) -> None:
|
||||
"""«Сделка от DD.MM.YYYY» в движении денег — расчёт по уже учтённой сделке."""
|
||||
assert not [e for e in full.events if (e.description or "").startswith("Сделка от")]
|
||||
|
||||
|
||||
def test_transfer_from_another_agreement_keeps_its_counterparty(full: ParsedReport) -> None:
|
||||
"""Перевод с другого договора Сбера — deposit, но с сохранённым источником.
|
||||
|
||||
Без `internal_transfer_from` эти деньги навсегда останутся внешним потоком: заведи
|
||||
пользователь второй договор счётом, XIRR увидел бы пополнение здесь и вывод там как два
|
||||
независимых события.
|
||||
"""
|
||||
transfers = [
|
||||
e
|
||||
for e in of_kind(full, EventKind.deposit)
|
||||
if e.meta.get("internal_transfer_from") == OTHER_AGREEMENT
|
||||
]
|
||||
assert len(transfers) == 2
|
||||
assert sum((e.amount for e in transfers), ZERO) == Decimal("3458.88") + Decimal("10551.18")
|
||||
|
||||
|
||||
def test_missing_coupon_section_is_reported_not_assumed(full: ParsedReport) -> None:
|
||||
"""Раздела «Купонный доход» в этих отчётах нет — это факт для data quality, не молчание."""
|
||||
assert any("Купонный доход" in w for w in full.warnings)
|
||||
|
||||
|
||||
# --- 9. деньги --------------------------------------------------------------------------------
|
||||
|
||||
|
||||
def numbers(report: ParsedReport) -> list[object]:
|
||||
values: list[object] = []
|
||||
for e in report.events:
|
||||
values += [e.amount, e.quantity, e.price, e.fee, e.tax, e.accrued_interest]
|
||||
for p in report.positions_end:
|
||||
values += [p.qty, p.price, p.market_value, p.accrued_interest]
|
||||
values += [c.balance for c in report.cash_end]
|
||||
return values
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["full", "august"])
|
||||
def test_no_floats_anywhere(name: str, request: pytest.FixtureRequest) -> None:
|
||||
"""Деньги — только Decimal (AGENTS.md). Один float здесь означает потерю копеек ниже."""
|
||||
report: ParsedReport = request.getfixturevalue(name)
|
||||
assert not [v for v in numbers(report) if isinstance(v, float)]
|
||||
assert all(isinstance(v, Decimal) for v in numbers(report) if v is not None)
|
||||
|
||||
|
||||
def test_every_event_carries_currency_and_traceability(full: ParsedReport) -> None:
|
||||
for event in full.events:
|
||||
assert event.currency == "RUB"
|
||||
assert event.raw_line_no > 0
|
||||
assert event.meta.get("section")
|
||||
@@ -0,0 +1,410 @@
|
||||
"""VTB xlsx report parser against the two anonymised fixtures.
|
||||
|
||||
The fixtures are the same account rendered twice: the whole life of the IIS (12.02–17.09) and
|
||||
the single month of August, which is a strict subset of it. That overlap is the point — it is
|
||||
exactly the case the dedupe key exists for, so the deal both files contain must produce one
|
||||
key, and the monthly file must not invent a trade the full one does not have.
|
||||
|
||||
Every number asserted here is taken from «Сводная информация по субсчёту Клиента», the
|
||||
broker's own totals block: the parser is trusted only insofar as what it extracted adds up to
|
||||
what VTB printed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import zipfile
|
||||
from collections import Counter
|
||||
from dataclasses import fields, is_dataclass
|
||||
from decimal import Decimal
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from openpyxl import load_workbook
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import ParsedReport
|
||||
from fintracker.sources.reports.vtb import common
|
||||
from fintracker.sources.reports.vtb.xlsx import VtbXlsxParser
|
||||
|
||||
FIXTURES = Path(__file__).resolve().parents[1] / "fixtures" / "reports"
|
||||
FULL = FIXTURES / "vtb" / "reportd0235f1e-55ac-fdc8-5c7b-279e4fe26e2b.xlsx"
|
||||
AUGUST = FIXTURES / "vtb" / "report8ef8347a-2deb-e880-959a-d9a8ca12f735.xlsx"
|
||||
|
||||
#: The August deal, as printed in both files: «№ сделки» B17476399721, 33 EQMX at 122.9.
|
||||
AUGUST_TRADE_NO = "B17476399721"
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def parser() -> VtbXlsxParser:
|
||||
return VtbXlsxParser()
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def full(parser: VtbXlsxParser) -> ParsedReport:
|
||||
return parser.parse(FULL.read_bytes(), FULL.name)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def august(parser: VtbXlsxParser) -> ParsedReport:
|
||||
return parser.parse(AUGUST.read_bytes(), AUGUST.name)
|
||||
|
||||
|
||||
def trades(report: ParsedReport) -> list:
|
||||
return [event for event in report.events if event.kind in {EventKind.buy, EventKind.sell}]
|
||||
|
||||
|
||||
def count_security_movement_rows(path: Path) -> int:
|
||||
"""Rows of «Движение ценных бумаг» counted straight off the sheet.
|
||||
|
||||
Deliberately independent of the parser: the section is the settlement side of the same
|
||||
deals, so its row count is an outside check that no trade was dropped or doubled.
|
||||
"""
|
||||
workbook = load_workbook(io.BytesIO(path.read_bytes()), read_only=True, data_only=True)
|
||||
sheet = workbook["brokerage_report"]
|
||||
sheet.reset_dimensions()
|
||||
inside = False
|
||||
rows = 0
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
label = common.normalize_text(row[1] if len(row) > 1 else None)
|
||||
if label == "движение ценных бумаг":
|
||||
inside = True
|
||||
continue
|
||||
if not inside:
|
||||
continue
|
||||
if label in {"заключенные в отчетном периоде сделки с ценными бумагами"}:
|
||||
break
|
||||
if "расчеты по заключенным сделкам" in common.normalize_text(
|
||||
row[16] if len(row) > 16 else None
|
||||
):
|
||||
rows += 1
|
||||
workbook.close()
|
||||
return rows
|
||||
|
||||
|
||||
# -- 1. sniff ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_sniff_accepts_both_vtb_files(parser: VtbXlsxParser) -> None:
|
||||
for path in (FULL, AUGUST):
|
||||
assert parser.sniff(path.read_bytes(), path.name) is True
|
||||
|
||||
|
||||
def test_sniff_rejects_other_brokers(parser: VtbXlsxParser) -> None:
|
||||
others = list((FIXTURES / "sber").glob("*.html")) + list((FIXTURES / "snowball").glob("*.csv"))
|
||||
assert others, "фикстуры других брокеров не найдены — проверка бессмысленна"
|
||||
for path in others:
|
||||
assert parser.sniff(path.read_bytes(), path.name) is False
|
||||
|
||||
|
||||
def test_sniff_never_raises_on_garbage(parser: VtbXlsxParser) -> None:
|
||||
"""A registry walks every parser over every upload, so a False must never be an
|
||||
exception — including for a zip that merely looks like a workbook."""
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w") as archive:
|
||||
archive.writestr("xl/workbook.xml", "<workbook/>")
|
||||
for payload in (b"", b"not a zip", b"PK\x03\x04broken", buffer.getvalue()):
|
||||
assert parser.sniff(payload, "whatever.xlsx") is False
|
||||
|
||||
|
||||
def test_sniff_ignores_the_file_name(parser: VtbXlsxParser) -> None:
|
||||
"""The name is a GUID with no format in it; content decides."""
|
||||
assert parser.sniff(FULL.read_bytes(), "totally-unrelated.bin") is True
|
||||
|
||||
|
||||
# -- 2. the full report ------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_report_header(full: ParsedReport) -> None:
|
||||
assert full.broker == "vtb"
|
||||
assert full.account_external_id == "77R593"
|
||||
assert (full.period_from.isoformat(), full.period_to.isoformat()) == (
|
||||
"2026-02-12",
|
||||
"2026-09-17",
|
||||
)
|
||||
assert full.meta["iis_opened_at"] == "2026-02-12"
|
||||
|
||||
|
||||
def test_full_report_trades_come_only_from_concluded_section(full: ParsedReport) -> None:
|
||||
"""«Завершенные» repeats «Заключенные» with the same deal numbers, and «Движение ценных
|
||||
бумаг» settles those same deals — so ten deals must yield ten events, not twenty or
|
||||
thirty."""
|
||||
numbers = [event.trade_no for event in trades(full)]
|
||||
assert len(numbers) == len(set(numbers)) == count_security_movement_rows(FULL) == 10
|
||||
|
||||
|
||||
def test_full_report_parses_without_warnings(full: ParsedReport) -> None:
|
||||
assert full.warnings == []
|
||||
|
||||
|
||||
# -- 3. money ----------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_report_deposits_and_fees(full: ParsedReport) -> None:
|
||||
deposits = [event for event in full.events if event.kind is EventKind.deposit]
|
||||
assert sum(event.amount for event in deposits) == Decimal("26100")
|
||||
assert all(event.amount > 0 for event in deposits), "пополнение увеличивает счёт"
|
||||
|
||||
# The per-deal commissions add up to «Вознаграждение брокера» of the summary block to the
|
||||
# kopeck, which is why they are capitalised into the deals instead of being emitted
|
||||
# separately: `ledger/lots.py` already puts `fee` into the cost of the lot.
|
||||
assert sum((event.fee or Decimal(0)) for event in full.events) == Decimal("3.64")
|
||||
assert Decimal(full.meta["summary"]["вознаграждение брокера"]) == Decimal("-3.64")
|
||||
assert not [event for event in full.events if event.kind is EventKind.commission]
|
||||
|
||||
|
||||
def test_full_report_signs(full: ParsedReport) -> None:
|
||||
for event in trades(full):
|
||||
assert event.quantity is not None
|
||||
if event.kind is EventKind.buy:
|
||||
assert event.quantity > 0 and event.amount < 0
|
||||
else:
|
||||
assert event.quantity < 0 and event.amount > 0
|
||||
|
||||
|
||||
def test_settlement_saldo_is_not_emitted_twice(full: ParsedReport) -> None:
|
||||
"""«Сальдо расчетов по сделкам с ценными бумагами» is the money leg of the trades; the
|
||||
cash section must contribute deposits only."""
|
||||
cash_kinds = Counter(event.kind for event in full.events if event.meta.get("section") == "cash")
|
||||
assert cash_kinds == Counter({EventKind.deposit: 8})
|
||||
|
||||
|
||||
def test_closing_cash_follows_from_the_events(full: ParsedReport) -> None:
|
||||
"""The whole-file check: opening + everything emitted = closing, as VTB states it."""
|
||||
summary = full.meta["summary"]
|
||||
opening = Decimal(summary["входящий остаток денежных средств"])
|
||||
closing = Decimal(summary["исходящий остаток денежных средств"])
|
||||
assert opening + sum(event.amount for event in full.events) == closing == Decimal("120.99")
|
||||
|
||||
|
||||
# -- 4. positions and cash balances -------------------------------------------------------
|
||||
|
||||
|
||||
def test_full_report_positions_end(full: ParsedReport) -> None:
|
||||
positions = {position.instrument.isin: position for position in full.positions_end}
|
||||
assert set(positions) == {"RU000A101EJ5", "RU000A1014L8"}
|
||||
assert positions["RU000A101EJ5"].qty == Decimal("197")
|
||||
assert positions["RU000A1014L8"].qty == Decimal("0")
|
||||
eqmx = positions["RU000A101EJ5"].instrument
|
||||
assert (eqmx.ticker, eqmx.asset_class_hint) == ("EQMX", "etf")
|
||||
assert eqmx.name == "EQMX ETF"
|
||||
assert eqmx.meta["reg_number"] == "3965"
|
||||
assert positions["RU000A101EJ5"].price == Decimal("124.653")
|
||||
|
||||
|
||||
def test_currency_is_normalised_to_iso(full: ParsedReport) -> None:
|
||||
"""VTB prints the pre-ISO `RUR`; the ledger stores three-letter ISO codes only."""
|
||||
assert [(cash.currency, cash.balance) for cash in full.cash_end] == [("RUB", Decimal("120.99"))]
|
||||
codes = {cash.currency for cash in full.cash_end}
|
||||
codes |= {position.currency for position in full.positions_end}
|
||||
codes |= {event.currency for event in full.events}
|
||||
codes |= {event.price_currency for event in trades(full)}
|
||||
assert codes == {"RUB"}
|
||||
|
||||
|
||||
def test_instrument_hint_reaches_trade_events(full: ParsedReport) -> None:
|
||||
"""Only the positions table names the asset class («ПАЙ»); the trades table does not, so
|
||||
the hint is carried over by ISIN."""
|
||||
assert {ref.asset_class_hint for ref in full.instruments} == {"etf"}
|
||||
assert all(
|
||||
event.instrument is not None and event.instrument.asset_class_hint == "etf"
|
||||
for event in trades(full)
|
||||
)
|
||||
|
||||
|
||||
# -- 5. the monthly report and the cross-file dedupe key ----------------------------------
|
||||
|
||||
|
||||
def test_august_report(august: ParsedReport) -> None:
|
||||
assert (august.period_from.isoformat(), august.period_to.isoformat()) == (
|
||||
"2026-08-01",
|
||||
"2026-08-31",
|
||||
)
|
||||
assert august.account_external_id == "77R593"
|
||||
assert august.warnings == []
|
||||
|
||||
deals = trades(august)
|
||||
assert len(deals) == 1
|
||||
deal = deals[0]
|
||||
assert deal.kind is EventKind.buy
|
||||
assert deal.instrument is not None and deal.instrument.isin == "RU000A101EJ5"
|
||||
assert deal.quantity == Decimal("33")
|
||||
assert deal.price == Decimal("122.9")
|
||||
assert deal.trade_no == AUGUST_TRADE_NO
|
||||
# 33 * 122.9 = 4055.70 plus the 0.81 commission capitalised into the deal.
|
||||
assert deal.amount == Decimal("-4056.51")
|
||||
assert deal.fee == Decimal("0.81")
|
||||
|
||||
|
||||
def test_same_deal_has_one_dedupe_key_in_both_reports(
|
||||
full: ParsedReport, august: ParsedReport
|
||||
) -> None:
|
||||
"""The overlapping-period case from plan §1.6 A: re-importing August after the full
|
||||
report must upsert, not duplicate."""
|
||||
keys = {}
|
||||
for report in (full, august):
|
||||
for event in trades(report):
|
||||
if event.trade_no == AUGUST_TRADE_NO:
|
||||
keys[report.period_to] = common.dedupe_key(report.account_external_id, event)
|
||||
assert len(keys) == 2
|
||||
assert len(set(keys.values())) == 1
|
||||
|
||||
|
||||
def test_dedupe_key_uses_the_brokers_own_deal_number(full: ParsedReport) -> None:
|
||||
"""«№ сделки» (Z), not «№ сделки у организатора торгов» (AC).
|
||||
|
||||
The key is `sha1(broker | account | trade_no)` and is therefore already unique within the
|
||||
broker; what it needs is the number VTB prints in every rendering of its own report. The
|
||||
organiser's number is a different string (`17476399721` against `B17476399721`) and is
|
||||
absent for over-the-counter deals in some formats.
|
||||
"""
|
||||
deal = next(event for event in trades(full) if event.trade_no == AUGUST_TRADE_NO)
|
||||
assert deal.meta["exchange_trade_no"] == "17476399721"
|
||||
assert common.dedupe_key("77R593", deal) != common.dedupe_key(
|
||||
"77R593",
|
||||
type(deal)(
|
||||
**{
|
||||
**{field.name: getattr(deal, field.name) for field in fields(deal)},
|
||||
"trade_no": deal.meta["exchange_trade_no"],
|
||||
}
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def test_cash_rows_get_distinct_fingerprint_keys(full: ParsedReport) -> None:
|
||||
"""Deposits carry no deal number and fall back to a fingerprint; two 4 000 ₽ top-ups in
|
||||
different months must still differ."""
|
||||
keys = [
|
||||
common.dedupe_key(full.account_external_id, event)
|
||||
for event in full.events
|
||||
if event.kind is EventKind.deposit
|
||||
]
|
||||
assert len(set(keys)) == len(keys) == 8
|
||||
|
||||
|
||||
# -- 6. concluded vs completed ------------------------------------------------------------
|
||||
|
||||
|
||||
def test_concluded_and_completed_deal_numbers_agree(full: ParsedReport) -> None:
|
||||
"""Asserted through the parser's own warning channel: a divergence is reported, and for a
|
||||
fully settled period there is none."""
|
||||
assert not [warning for warning in full.warnings if "Завершенных" in warning]
|
||||
parsed = VtbXlsxParser().parse(FULL.read_bytes(), FULL.name)
|
||||
concluded = {event.trade_no for event in trades(parsed)}
|
||||
assert concluded == set(_completed_deal_numbers(FULL))
|
||||
|
||||
|
||||
def _completed_deal_numbers(path: Path) -> list[str]:
|
||||
"""«№ сделки» of «Завершенные…» read straight off the sheet, bypassing the parser."""
|
||||
workbook = load_workbook(io.BytesIO(path.read_bytes()), read_only=True, data_only=True)
|
||||
sheet = workbook["brokerage_report"]
|
||||
sheet.reset_dimensions()
|
||||
inside = False
|
||||
numbers: list[str] = []
|
||||
for row in sheet.iter_rows(values_only=True):
|
||||
label = common.normalize_text(row[1] if len(row) > 1 else None)
|
||||
if label.startswith("завершенные в отчетном периоде сделки с ценными бумагами"):
|
||||
inside = True
|
||||
continue
|
||||
if not inside:
|
||||
continue
|
||||
value = row[25] if len(row) > 25 else None # column Z
|
||||
if value and str(value).strip() and common.normalize_text(value) != "№ сделки":
|
||||
numbers.append(str(value).strip())
|
||||
workbook.close()
|
||||
return numbers
|
||||
|
||||
|
||||
# -- 7. no floats anywhere ----------------------------------------------------------------
|
||||
|
||||
|
||||
def _floats(value: object, path: str = "") -> list[str]:
|
||||
if isinstance(value, bool) or value is None:
|
||||
return []
|
||||
if isinstance(value, float):
|
||||
return [path]
|
||||
if is_dataclass(value) and not isinstance(value, type):
|
||||
found: list[str] = []
|
||||
for field in fields(value):
|
||||
found += _floats(getattr(value, field.name), f"{path}.{field.name}")
|
||||
return found
|
||||
if isinstance(value, dict):
|
||||
return [p for key, item in value.items() for p in _floats(item, f"{path}[{key}]")]
|
||||
if isinstance(value, (list, tuple, set)):
|
||||
return [p for index, item in enumerate(value) for p in _floats(item, f"{path}[{index}]")]
|
||||
return []
|
||||
|
||||
|
||||
@pytest.mark.parametrize("name", ["full", "august"])
|
||||
def test_no_floats_in_output(name: str, request: pytest.FixtureRequest) -> None:
|
||||
"""openpyxl hands back floats for every numeric cell; none of them may survive into the
|
||||
result, or a kopeck disappears into binary rounding on the way to `NUMERIC(24,10)`."""
|
||||
report: ParsedReport = request.getfixturevalue(name)
|
||||
assert _floats(report, name) == []
|
||||
|
||||
|
||||
# -- 8. totals against the summary block --------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize(("name", "expected"), [("full", "-25975.37"), ("august", "-4055.7")])
|
||||
def test_trade_sums_match_the_settlement_saldo(
|
||||
name: str, expected: str, request: pytest.FixtureRequest
|
||||
) -> None:
|
||||
"""Σ «Сумма сделки» (column M) of the purchases minus the sales equals «Сальдо расчетов по
|
||||
сделкам с ценными бумагами» in the summary.
|
||||
|
||||
The commission has to be taken back out of `amount` to make the comparison: VTB's saldo is
|
||||
the money of the deals themselves, while `event.amount` carries the fee inside it by the
|
||||
ledger's convention. The two agree exactly once that is done — the fee lives in the same
|
||||
file as its own «Вознаграждение брокера» line, and it is that line, not the saldo, that
|
||||
accounts for it.
|
||||
"""
|
||||
report: ParsedReport = request.getfixturevalue(name)
|
||||
gross = Decimal(0)
|
||||
for event in trades(report):
|
||||
fee = event.fee or Decimal(0)
|
||||
gross += event.amount + fee if event.kind is EventKind.buy else event.amount - fee
|
||||
assert gross == Decimal(expected)
|
||||
assert gross == Decimal(report.meta["summary"]["сальдо расчетов по сделкам с ценными бумагами"])
|
||||
|
||||
|
||||
def test_position_quantities_match_the_settled_movements(full: ParsedReport) -> None:
|
||||
"""Closing position = what the trades did, since the account opened empty."""
|
||||
by_isin: dict[str, Decimal] = {}
|
||||
for event in trades(full):
|
||||
assert event.instrument is not None and event.instrument.isin
|
||||
assert event.quantity is not None
|
||||
by_isin[event.instrument.isin] = (
|
||||
by_isin.get(event.instrument.isin, Decimal(0)) + event.quantity
|
||||
)
|
||||
for position in full.positions_end:
|
||||
assert position.instrument.isin is not None
|
||||
assert by_isin[position.instrument.isin] == position.qty
|
||||
|
||||
|
||||
# -- parse errors --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_unreadable_file_raises_parse_error(parser: VtbXlsxParser) -> None:
|
||||
from fintracker.sources.reports.base import ParseError
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse(b"PK\x03\x04 definitely not a workbook", "report.xlsx")
|
||||
|
||||
|
||||
def test_workbook_without_a_period_raises_parse_error(parser: VtbXlsxParser) -> None:
|
||||
"""A file we recognise but cannot place on a timeline must fail loudly rather than import
|
||||
as an empty report."""
|
||||
from openpyxl import Workbook
|
||||
|
||||
from fintracker.sources.reports.base import ParseError
|
||||
|
||||
workbook = Workbook()
|
||||
sheet = workbook.worksheets[0]
|
||||
sheet.title = "brokerage_report"
|
||||
sheet["D1"] = "Отчет Банка ВТБ (ПАО) без периода"
|
||||
buffer = io.BytesIO()
|
||||
workbook.save(buffer)
|
||||
|
||||
with pytest.raises(ParseError):
|
||||
parser.parse(buffer.getvalue(), "report.xlsx")
|
||||
Reference in New Issue
Block a user