feat(reports): протокол и парсеры отчётов Сбера, ВТБ и Snowball CSV
sources/reports/base.py — контракт ReportParser (плагины: sniff/parse, чистые функции без БД). registry.py выбирает парсер по содержимому файла, CSV последним: он узнаёт файл по набору колонок и иначе перехватил бы чужой формат. Комиссия капитализируется в сделку, отдельным событием не эмитится: и Сбер, и ВТБ печатают её дважды — колонками в сделках и строками в движении денег, суммы совпадают, второе прочтение задвоило бы её. Расчётные строки («Сделка от …», «Сальдо расчетов по сделкам») не эмитятся — это денежные ноги уже учтённых сделок. У Сбера таблица «Информация о зачислениях на ИИС» кумулятивна за календарный год и в леджер не идёт, иначе два пересекающихся отчёта задвоили бы пополнения. CSV Snowball сводный по всем брокерам — годится как сверка потоков и сделок на уровне портфеля, но не позиций по счетам; CUSTOM_HOLDING_PRICE не событие, а цена — уходит в meta для price_manual. Обезличивание — anonymize.py + scripts/anonymize_reports.py, секреты собираются по всему корпусу отчётов разом (Snowball цитирует номер договора Сбера в примечании к переводу). test_fixtures_anonymized.py падает, если в tests/fixtures/reports/ вне raw/ найдётся ИНН, ФИО или номер счёта — и по форме (работает в CI без raw/), и по фактическому содержимому raw/, когда оно на месте.
This commit is contained in:
Dmitry
@@ -0,0 +1,32 @@
|
||||
"""Broker report parsers (plan §2 «Интерфейсы плагинов», фаза 3).
|
||||
|
||||
A parser turns the bytes of one uploaded file into a `ParsedReport`: a period, a broker
|
||||
account id, a flat list of `BrokerEvent`, the closing positions and cash the report itself
|
||||
states, and the instruments it mentions. Nothing here touches the database — the whole
|
||||
package is importable and testable from a fixture alone, which is the point: report formats
|
||||
change without warning, and a format regression must be reproducible from one file.
|
||||
"""
|
||||
|
||||
from fintracker.sources.reports.base import (
|
||||
BrokerEvent,
|
||||
CashEnd,
|
||||
InstrumentRef,
|
||||
ParsedReport,
|
||||
ParseError,
|
||||
PositionEnd,
|
||||
ReportParser,
|
||||
fingerprint_key,
|
||||
trade_dedupe_key,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"BrokerEvent",
|
||||
"CashEnd",
|
||||
"InstrumentRef",
|
||||
"ParseError",
|
||||
"ParsedReport",
|
||||
"PositionEnd",
|
||||
"ReportParser",
|
||||
"fingerprint_key",
|
||||
"trade_dedupe_key",
|
||||
]
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Turning a real broker report into a committable fixture (AGENTS.md: только обезличенные).
|
||||
|
||||
A broker report is a personal document: it prints the investor's full name, their taxpayer
|
||||
number, the agreement and sub-account numbers, and the settlement account. None of that may
|
||||
reach git, while everything a parser is tested on — dates, amounts, quantities, tickers,
|
||||
ISINs, deal numbers, section layout — must survive untouched, or the fixture stops testing
|
||||
the parser.
|
||||
|
||||
Two rules make this reproducible rather than a one-off scrub:
|
||||
|
||||
* **Nothing secret is written down here.** The module *finds* the personal data in the file
|
||||
by the labels the broker itself prints («Инвестор:», «Клиент:», «ИНН:», «№ субсчета:»),
|
||||
so this source file is safe to commit and the same raw file always yields the same
|
||||
fixture.
|
||||
* **Replacements preserve shape.** An account number keeps its length and its
|
||||
letter/digit layout, because the parser reads it as an identifier and a fixture with a
|
||||
differently-shaped id would not exercise the same code path. The substitute is derived
|
||||
from a hash of the original, so it is stable across runs but tells you nothing about it.
|
||||
|
||||
`tests/test_fixtures_anonymized.py` is the enforcement side: it re-derives the secrets from
|
||||
`raw/` (when present) and fails if any of them appears in a committed fixture.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import io
|
||||
import re
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
from dataclasses import dataclass, field
|
||||
|
||||
DIGITS = "0123456789"
|
||||
UPPER = "АБВГДЕЖЗИКЛМНОПРСТУФХЦЧШЩЭЮЯ"
|
||||
LATIN = "ABCDEFGHIJKLMNPQRSTUVWXYZ"
|
||||
|
||||
#: A fixture may legitimately contain these — they are the substitutes this module produces.
|
||||
FAKE_SURNAME = "Иванов"
|
||||
FAKE_NAME = "Иван"
|
||||
FAKE_PATRONYMIC = "Иванович"
|
||||
|
||||
# --- what counts as personal data -------------------------------------------------------
|
||||
|
||||
#: Individual taxpayer number: 12 digits. A legal entity's is 10 and is public information
|
||||
#: (the clearing house's INN is printed in every VTB report), so only 12 is scrubbed.
|
||||
#: The lookarounds exclude a decimal point on either side — an xlsx stores a timestamp as
|
||||
#: `46247.480069444515`, and the fractional part is twelve digits that mean nothing.
|
||||
RE_INN = re.compile(r"(?<![\d.])\d{12}(?![\d.])")
|
||||
|
||||
#: Settlement / depo account: 20 digits.
|
||||
RE_ACCOUNT_20 = re.compile(r"(?<![\d.])\d{20}(?![\d.])")
|
||||
|
||||
#: Substitutes for the two numeric identifiers are sentinels, not hashes: a hash of a
|
||||
#: 12-digit INN is another 12-digit number, and the fixture check could not then tell a
|
||||
#: leak from a replacement. Six leading zeros never occur in a real INN or account number,
|
||||
#: so `RE_PLACEHOLDER_NUMBER` is a safe allow-list for the test.
|
||||
RE_PLACEHOLDER_NUMBER = re.compile(r"^0{6,}\d*$")
|
||||
|
||||
#: «Петров И. С.» — surname plus initials, in any spacing. VTB prints the client this way,
|
||||
#: and its own back-office employee too, so both are caught.
|
||||
RE_SHORT_NAME = re.compile(r"\b([А-ЯЁ][а-яё]{2,})\s+([А-ЯЁ])\.\s?([А-ЯЁ])\.")
|
||||
|
||||
#: «ПЕТРОВ ИВАН СЕРГЕЕВИЧ» — the all-caps form Sber prints after «Инвестор:».
|
||||
RE_CAPS_FIO = re.compile(r"\b[А-ЯЁ]{3,}\s+[А-ЯЁ]{3,}\s+[А-ЯЁ]{3,}\b")
|
||||
|
||||
#: Labels after which a broker prints the agreement / sub-account id.
|
||||
ACCOUNT_ANCHORS = (
|
||||
re.compile(
|
||||
r"(?:индивидуального инвестиционного счета|Договор[^\n<]{0,60}?)\s+([0-9A-Z]{5,12})\s+от"
|
||||
),
|
||||
re.compile(r"№ и дата Соглашения[^0-9A-Z]{0,40}([0-9A-Z]{5,12})\b"),
|
||||
re.compile(r"№\s*субсчета[^0-9A-Z]{0,40}([0-9A-Z]{5,12})\b"),
|
||||
re.compile(r"Субпозиция\s*№\s*([0-9A-Z]{5,12})\b"),
|
||||
re.compile(r"с договора\s+([0-9A-Z]{5,12})\b"),
|
||||
re.compile(r"на договор\s+([0-9A-Z]{5,12})\b"),
|
||||
)
|
||||
|
||||
#: Tokens that match an anchor's shape but are not identifiers of this client.
|
||||
ACCOUNT_STOPWORDS = frozenset({"UCAF", "RUB", "USD", "EUR"})
|
||||
|
||||
|
||||
@dataclass
|
||||
class Secrets:
|
||||
"""What was found in one report, and what each item is replaced with."""
|
||||
|
||||
replacements: dict[str, str] = field(default_factory=dict)
|
||||
|
||||
def add(self, original: str, substitute: str) -> None:
|
||||
if original and original not in self.replacements:
|
||||
self.replacements[original] = substitute
|
||||
|
||||
def apply(self, text: str) -> str:
|
||||
# Longest first: an agreement id may be a substring of a longer one.
|
||||
for original in sorted(self.replacements, key=len, reverse=True):
|
||||
text = text.replace(original, self.replacements[original])
|
||||
return text
|
||||
|
||||
@property
|
||||
def originals(self) -> list[str]:
|
||||
return list(self.replacements)
|
||||
|
||||
|
||||
def _shaped_substitute(value: str) -> str:
|
||||
"""A stable stand-in with the same length and letter/digit layout as `value`."""
|
||||
digest = hashlib.sha256(value.encode("utf-8")).digest()
|
||||
out: list[str] = []
|
||||
for i, ch in enumerate(value):
|
||||
byte = digest[i % len(digest)]
|
||||
if ch.isdigit():
|
||||
out.append(DIGITS[byte % 10])
|
||||
elif ch.isalpha() and ch.isupper():
|
||||
alphabet = UPPER if ch in UPPER or ord(ch) > 127 else LATIN
|
||||
out.append(alphabet[byte % len(alphabet)])
|
||||
elif ch.isalpha():
|
||||
out.append(LATIN[byte % len(LATIN)].lower())
|
||||
else:
|
||||
out.append(ch)
|
||||
return "".join(out)
|
||||
|
||||
|
||||
def find_secrets(text: str) -> Secrets:
|
||||
"""Everything personal in one report's text, with its replacement decided."""
|
||||
secrets = Secrets()
|
||||
|
||||
for match in RE_CAPS_FIO.finditer(text):
|
||||
fio = match.group(0)
|
||||
secrets.add(fio, f"{FAKE_SURNAME.upper()} {FAKE_NAME.upper()} {FAKE_PATRONYMIC.upper()}")
|
||||
|
||||
for match in RE_SHORT_NAME.finditer(text):
|
||||
secrets.add(match.group(0), f"{FAKE_SURNAME} {FAKE_NAME[0]}. {FAKE_PATRONYMIC[0]}.")
|
||||
|
||||
for pattern in (RE_INN, RE_ACCOUNT_20):
|
||||
for match in pattern.finditer(text):
|
||||
original = match.group(0)
|
||||
if RE_PLACEHOLDER_NUMBER.match(original):
|
||||
continue # already a substitute — re-running must be idempotent
|
||||
index = sum(1 for v in secrets.replacements.values() if len(v) == len(original))
|
||||
secrets.add(original, "0" * (len(original) - 2) + f"{index:02d}")
|
||||
|
||||
for pattern in ACCOUNT_ANCHORS:
|
||||
for match in pattern.finditer(text):
|
||||
token = match.group(1)
|
||||
if token in ACCOUNT_STOPWORDS or token.isalpha():
|
||||
continue
|
||||
secrets.add(token, _shaped_substitute(token))
|
||||
|
||||
return secrets
|
||||
|
||||
|
||||
# --- per-format rewriting ----------------------------------------------------------------
|
||||
|
||||
|
||||
def anonymize_text(
|
||||
data: bytes, encoding: str = "utf-8", secrets: Secrets | None = None
|
||||
) -> tuple[bytes, Secrets]:
|
||||
"""HTML and CSV: one decode, one pass, one encode."""
|
||||
text = data.decode(encoding)
|
||||
secrets = secrets if secrets is not None else find_secrets(text)
|
||||
return secrets.apply(text).encode(encoding), secrets
|
||||
|
||||
|
||||
#: Parts of an xlsx that can hold a visible string. Styles and calc chains cannot.
|
||||
XLSX_TEXT_PARTS = re.compile(r"^xl/(sharedStrings\.xml|worksheets/.*\.xml|comments\d*\.xml)$")
|
||||
|
||||
|
||||
def anonymize_xlsx(data: bytes, secrets: Secrets | None = None) -> tuple[bytes, Secrets]:
|
||||
"""Rewrite the strings inside a workbook, leaving every other part byte-identical.
|
||||
|
||||
Editing the zip directly rather than round-tripping through openpyxl is deliberate: a
|
||||
load/save cycle rewrites formatting, drops the parts openpyxl does not model, and would
|
||||
make the fixture differ from a real export in ways a parser might depend on.
|
||||
"""
|
||||
source = zipfile.ZipFile(io.BytesIO(data))
|
||||
texts = {
|
||||
name: source.read(name).decode("utf-8")
|
||||
for name in source.namelist()
|
||||
if XLSX_TEXT_PARTS.match(name)
|
||||
}
|
||||
secrets = secrets if secrets is not None else find_secrets("\n".join(texts.values()))
|
||||
|
||||
buffer = io.BytesIO()
|
||||
with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as out:
|
||||
for item in source.infolist():
|
||||
if item.filename in texts:
|
||||
out.writestr(item, secrets.apply(texts[item.filename]).encode("utf-8"))
|
||||
else:
|
||||
out.writestr(item, source.read(item.filename))
|
||||
return buffer.getvalue(), secrets
|
||||
|
||||
|
||||
def readable_text(data: bytes, filename: str) -> str:
|
||||
"""The visible text of a report, whatever its container — the input to `find_secrets`."""
|
||||
if filename.lower().endswith((".xlsx", ".xlsm")):
|
||||
archive = zipfile.ZipFile(io.BytesIO(data))
|
||||
return "\n".join(
|
||||
archive.read(name).decode("utf-8", "replace")
|
||||
for name in archive.namelist()
|
||||
if XLSX_TEXT_PARTS.match(name)
|
||||
)
|
||||
return data.decode("utf-8", "replace")
|
||||
|
||||
|
||||
def collect_secrets(files: Iterable[tuple[bytes, str]]) -> Secrets:
|
||||
"""One secret set for a whole batch of reports — the only safe way to scrub them.
|
||||
|
||||
Personal data crosses file boundaries: the Snowball CSV quotes the Sber agreement number
|
||||
inside a transfer's note, and nothing in the CSV itself marks it as an identifier. Each
|
||||
file scrubbed in isolation therefore leaks what another file named. Deriving the set once
|
||||
from every original and applying it to all of them closes that gap, and keeps one
|
||||
identifier mapped to one substitute across the whole fixture corpus.
|
||||
"""
|
||||
merged = Secrets()
|
||||
for data, filename in files:
|
||||
for original, substitute in find_secrets(
|
||||
readable_text(data, filename)
|
||||
).replacements.items():
|
||||
merged.add(original, substitute)
|
||||
return merged
|
||||
|
||||
|
||||
def anonymize(data: bytes, filename: str, secrets: Secrets | None = None) -> tuple[bytes, Secrets]:
|
||||
"""Dispatch on extension. Returns the clean bytes and what was removed."""
|
||||
lowered = filename.lower()
|
||||
if lowered.endswith((".xlsx", ".xlsm")):
|
||||
return anonymize_xlsx(data, secrets)
|
||||
if lowered.endswith(".csv"):
|
||||
return anonymize_text(data, "utf-8-sig", secrets)
|
||||
return anonymize_text(data, "utf-8", secrets)
|
||||
|
||||
|
||||
def anonymize_filename(filename: str, secrets: Secrets) -> str:
|
||||
"""Sber names its exports after the account number — the name leaks too."""
|
||||
return secrets.apply(filename)
|
||||
@@ -0,0 +1,231 @@
|
||||
"""The `ReportParser` contract and the value objects every parser produces (plan §2).
|
||||
|
||||
Why these types and not the ORM rows directly: a report is *evidence*, not truth. It names
|
||||
an instrument the way its own back office names it, states a period that may overlap another
|
||||
file, and carries closing balances that exist precisely so they can be compared against what
|
||||
the ledger derives. Handing `ingest.py` a plain dataclass keeps the parser free of the
|
||||
database — a format regression is then reproducible from one fixture and nothing else.
|
||||
|
||||
Sign conventions match `models/ledger.Event` exactly, so ingest never has to re-interpret:
|
||||
|
||||
* `quantity` is signed by its effect on the POSITION: + into it, - out of it.
|
||||
* `amount` is signed by its effect on the ACCOUNT's cash: + received, - paid.
|
||||
* `fee` and `tax` are positive and already reflected inside `amount`.
|
||||
|
||||
**Dedup keys.** Two report files routinely describe the same trade — overlapping periods, a
|
||||
re-export, the xlsx and the pdf of one month. A key must therefore be derived from the trade
|
||||
itself, never from the file. `trade_dedupe_key` is the good case: the broker's own deal
|
||||
number is stable across every rendering of it. `fingerprint_key` is the fallback for rows
|
||||
that have no number (a cash movement, a fee line): it hashes what actually identifies the
|
||||
row economically. The fallback is genuinely weaker — two identical purchases of the same
|
||||
paper at the same price on one day collapse into one key — so a parser that has a deal
|
||||
number must pass it, and a parser that has to fall back should add a discriminator
|
||||
(`seq`) that is stable for the same report content, not the line number of a particular
|
||||
rendering.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from typing import Any, Protocol, runtime_checkable
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
|
||||
|
||||
class ParseError(ValueError):
|
||||
"""The file is this parser's format but its content cannot be read.
|
||||
|
||||
Distinct from `sniff` returning False (wrong format, try another parser): raising this
|
||||
means the right parser gave up, and the import must fail loudly instead of silently
|
||||
producing an empty report.
|
||||
"""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class InstrumentRef:
|
||||
"""How a report names a security, before anything resolves it to an `instrument` row.
|
||||
|
||||
Every field is optional because reports differ in what they print: Sber gives ISIN and a
|
||||
ticker, VTB gives a name plus ISIN inside one string, a CSV may give a ticker only.
|
||||
Resolution order is the plan's (§1.3): ISIN → FIGI → tinvest_uid → (ticker, board) →
|
||||
alias. `source_key` is what an `instrument_alias` row would store when nothing else
|
||||
matches, and it is what a `pending_instrument` is keyed by — so it must be stable for
|
||||
the same paper across files.
|
||||
"""
|
||||
|
||||
isin: str | None = None
|
||||
ticker: str | None = None
|
||||
board: str | None = None
|
||||
name: str | None = None
|
||||
currency: str | None = None
|
||||
asset_class_hint: str | None = None
|
||||
"""share | bond | etf | fund | currency — a hint from the report's own wording, never
|
||||
authoritative: the user confirms the class when resolving a pending instrument."""
|
||||
source_key: str | None = None
|
||||
"""Stable key for `instrument_alias(source, source_key)`, e.g. 'ISIN:RU0009029540'."""
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
def key(self) -> str:
|
||||
"""The identity used for pending-instrument grouping and alias lookups."""
|
||||
if self.source_key:
|
||||
return self.source_key
|
||||
if self.isin:
|
||||
return f"ISIN:{self.isin}"
|
||||
if self.ticker and self.board:
|
||||
return f"TICKER:{self.ticker}/{self.board}"
|
||||
if self.ticker:
|
||||
return f"TICKER:{self.ticker}"
|
||||
return f"NAME:{self.name or ''}"
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class BrokerEvent:
|
||||
"""One ledger-bound row of a report, already in `event`'s sign conventions."""
|
||||
|
||||
kind: EventKind
|
||||
trade_date: date
|
||||
amount: Decimal
|
||||
"""Signed cash effect on the account: + received, - paid. Zero for a pure position move."""
|
||||
currency: str
|
||||
settle_date: date | None = None
|
||||
instrument: InstrumentRef | None = None
|
||||
quantity: Decimal | None = None
|
||||
price: Decimal | None = None
|
||||
price_currency: str | None = None
|
||||
fee: Decimal | None = None
|
||||
fee_currency: str | None = None
|
||||
tax: Decimal | None = None
|
||||
tax_currency: str | None = None
|
||||
accrued_interest: Decimal | None = None
|
||||
trade_no: str | None = None
|
||||
"""The broker's own deal number — the only stable identity across re-exports."""
|
||||
description: str | None = None
|
||||
raw_line_no: int = 0
|
||||
"""Where the row sat in the parsed file; stored on `raw_report_line` for traceability.
|
||||
Never part of a dedupe key: the same trade sits on different lines in different files."""
|
||||
seq: int = 0
|
||||
"""Discriminator for fingerprint keys when a report repeats an identical row legitimately
|
||||
(two equal purchases in one second). Must be derived from the row's position *within its
|
||||
own section*, so the same content yields the same value in any rendering of the report."""
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
"""Parser-specific extras carried to `event.meta`: split ratio, deal comment, section."""
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class PositionEnd:
|
||||
"""A closing position as the report states it — reconciliation input, never a ledger row."""
|
||||
|
||||
instrument: InstrumentRef
|
||||
qty: Decimal
|
||||
price: Decimal | None = None
|
||||
market_value: Decimal | None = None
|
||||
currency: str | None = None
|
||||
accrued_interest: Decimal | None = None
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CashEnd:
|
||||
"""A closing cash balance per currency, as the report states it."""
|
||||
|
||||
currency: str
|
||||
balance: Decimal
|
||||
blocked: Decimal | None = None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ParsedReport:
|
||||
"""Everything one file yields. `warnings` is the parser's own data-quality channel:
|
||||
a section it did not recognise, a total that did not add up, a row it skipped."""
|
||||
|
||||
broker: str
|
||||
"""sber | vtb | csv — matches `Broker` and the `report_*` source names."""
|
||||
account_external_id: str
|
||||
"""The broker's account/agreement number as printed, e.g. 'S930W42', '77R593'."""
|
||||
period_from: date
|
||||
period_to: date
|
||||
parser_version: str
|
||||
events: list[BrokerEvent] = field(default_factory=list)
|
||||
positions_end: list[PositionEnd] = field(default_factory=list)
|
||||
cash_end: list[CashEnd] = field(default_factory=list)
|
||||
instruments: list[InstrumentRef] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
meta: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@runtime_checkable
|
||||
class ReportParser(Protocol):
|
||||
"""Pure, database-free: bytes in, `ParsedReport` out."""
|
||||
|
||||
broker: str
|
||||
name: str
|
||||
"""Registry key and `event.source`, e.g. 'report_sber'."""
|
||||
formats: tuple[str, ...]
|
||||
"""Lowercase extensions this parser reads, e.g. ('html', 'htm')."""
|
||||
version: str
|
||||
"""Bumped whenever output changes for the same input; stored on `raw_report_file`."""
|
||||
|
||||
def sniff(self, data: bytes, filename: str) -> bool:
|
||||
"""True when this parser recognises the file. Cheap, and never raises."""
|
||||
...
|
||||
|
||||
def parse(self, data: bytes, filename: str) -> ParsedReport:
|
||||
"""Parse, or raise `ParseError` when the content is unreadable."""
|
||||
...
|
||||
|
||||
|
||||
def _sha1(parts: list[str]) -> str:
|
||||
return hashlib.sha1("|".join(parts).encode("utf-8")).hexdigest()
|
||||
|
||||
|
||||
def trade_dedupe_key(broker: str, account_external_id: str, trade_no: str) -> str:
|
||||
"""sha1(broker|account|trade_no) — the plan's §1.6 A key for a numbered deal."""
|
||||
return _sha1([broker, account_external_id, trade_no])
|
||||
|
||||
|
||||
def fingerprint_key(
|
||||
broker: str,
|
||||
account_external_id: str,
|
||||
kind: EventKind | str,
|
||||
instrument_key: str,
|
||||
trade_date: date,
|
||||
quantity: Decimal | None,
|
||||
price: Decimal | None,
|
||||
currency: str,
|
||||
*,
|
||||
amount: Decimal | None = None,
|
||||
seq: int = 0,
|
||||
) -> str:
|
||||
"""Fallback key for a row with no deal number (plan §1.6 A).
|
||||
|
||||
`amount` joins the plan's list because a report's cash section is full of rows that share
|
||||
everything else — two 4 000 ₽ top-ups in one month differ by nothing but the amount and
|
||||
the date, and a fee line has neither quantity nor price. `seq` breaks the remaining ties.
|
||||
"""
|
||||
kind_value = kind.value if isinstance(kind, EventKind) else str(kind)
|
||||
return _sha1(
|
||||
[
|
||||
broker,
|
||||
account_external_id,
|
||||
kind_value,
|
||||
instrument_key,
|
||||
trade_date.isoformat(),
|
||||
_num(quantity),
|
||||
_num(price),
|
||||
currency,
|
||||
_num(amount),
|
||||
str(seq),
|
||||
]
|
||||
)
|
||||
|
||||
|
||||
def _num(value: Decimal | None) -> str:
|
||||
"""Normalised so 1.50 and 1.5 hash alike — a re-export may print either."""
|
||||
if value is None:
|
||||
return ""
|
||||
normalized = value.normalize()
|
||||
if normalized == 0:
|
||||
return "0"
|
||||
return format(normalized, "f")
|
||||
@@ -0,0 +1,12 @@
|
||||
"""Universal CSV report parser (plan §2, фаза 3).
|
||||
|
||||
The format is Snowball Income's own export — the one file that covers *every* brokerage
|
||||
account at once, which makes it the only independent cross-check of the whole ledger we
|
||||
can get without re-reading each broker's own statements. It is therefore parsed as a
|
||||
generic "event CSV" rather than as a Snowball-branded format: recognition is by the header
|
||||
columns, so any future export with the same column set is read by the same code.
|
||||
"""
|
||||
|
||||
from fintracker.sources.reports.csv_universal.parser import UniversalCsvParser
|
||||
|
||||
__all__ = ["UniversalCsvParser"]
|
||||
@@ -0,0 +1,791 @@
|
||||
"""A generic event CSV (Snowball Income's export) → `ParsedReport`.
|
||||
|
||||
The column set is fixed; what a column *means* is not. `Quantity` holds a number of shares
|
||||
on a BUY and a number of rubles on a DIVIDEND, `Price` holds a price on a trade and a split
|
||||
ratio on a SPLIT, and a FEE row carries its money in `FeeTax` with every other numeric
|
||||
column zero. So the parser is organised as one handler per event type rather than as a
|
||||
single row→event mapping: the shape of the row *is* the event type, and pretending
|
||||
otherwise is how a dividend of 131,44 ₽ becomes a position of 131,44 bonds.
|
||||
|
||||
Contracts with the rest of the import pipeline, all decided here and documented in
|
||||
`docs/ai/csv-import.md`:
|
||||
|
||||
* **`CUSTOM_HOLDING_PRICE` is a price, not an event.** It is a user-entered quote for a
|
||||
paper the exchange does not quote (in the fixture: SIBN6P4, a bond held off-market), and
|
||||
its home is `price_manual`. `base.py` has nowhere to put a price, and `base.py` is not
|
||||
ours to change, so the rows travel in `meta["manual_prices"]` as
|
||||
`{"instrument_key", "d", "price", "currency"}` and the instruments they name are also
|
||||
listed in `instruments`, so `ingest.py` resolves them through the ordinary pending-
|
||||
instrument flow before writing the prices.
|
||||
|
||||
* **`CUSTOM_HOLDING_SETTINGS` describes an instrument, not an event.** Its `Note` is JSON
|
||||
with every `"` rewritten to `@*@` (so the CSV never has to escape quotes). Undo that,
|
||||
read `Holding.Description`/`Holding.Currency`/`Holding.Sector`, and emit an
|
||||
`InstrumentRef` — this is the only place a custom holding gets a human name. Malformed
|
||||
JSON is a `warning`, never an exception: one broken settings row must not cost the user
|
||||
500 good trades.
|
||||
|
||||
* **A `CASH_IN`/`CASH_OUT` whose `Note` says it balances the account is not a flow.**
|
||||
Snowball inserts these itself when the broker's stated cash disagrees with the cash its
|
||||
own replay produces. Emitting them as deposit/withdrawal would feed XIRR an external flow
|
||||
that never crossed the portfolio boundary. They go to `warnings` and to
|
||||
`meta["balance_adjustments"]`. Detection is by the note's wording, never by the size of
|
||||
the amount: the fixture has one such correction of 1017,84 ₽ next to genuine withdrawals
|
||||
of 32 ₽, so a threshold would misclassify both.
|
||||
|
||||
* **A `DIVIDEND` on a bond is a coupon, and the parser cannot prove it.** Asset class lives
|
||||
in the database and the parser has none. So the kind stays `dividend` and the suspicion
|
||||
travels as `meta["payout_hint"] = "coupon"` whenever the symbol looks like a bond (a
|
||||
valid ISIN, or an OFZ secid `SU…RMFS…`). `ledger/ingest.py` reclassifies to
|
||||
`EventKind.coupon` once the instrument resolves and its real `asset_class` is known; the
|
||||
hint is a shortcut for the resolver, not an answer.
|
||||
|
||||
* **The file names a portfolio, not an account.** It covers every broker at once — 49
|
||||
symbols from three brokers in one 545-row fixture — so `account_external_id` cannot be
|
||||
derived from it and is filled with the portfolio name out of the filename. A warning says
|
||||
so, because per plan §1.6 B an account has one `primary_event_source`: whatever account
|
||||
the user points this import at, most of these events will land as `status = shadow` and
|
||||
serve as reconciliation evidence rather than as the ledger itself.
|
||||
|
||||
* **An unknown event type is skipped with a warning, never mapped to `other`.** `other` is
|
||||
a silent bucket; a format that grew a new row type must be visible as a data-quality
|
||||
finding, not averaged into a metric.
|
||||
|
||||
**Dedup.** The format has no deal numbers, so every row gets `fingerprint_key`, and the
|
||||
whole weight of telling two real trades apart from one trade exported twice falls on `seq`.
|
||||
`seq` is the rank of the row's (time-of-day, note) among the distinct (time-of-day, note)
|
||||
pairs of all rows that share its fingerprint content. Consequences, both intended:
|
||||
|
||||
* two genuine 1 100 ₽ top-ups one second apart keep distinct keys, and a re-import of the
|
||||
same file reproduces those keys exactly, because the rank depends on content and clock
|
||||
time only — never on line numbers;
|
||||
* two rows identical down to the second *and* the note collapse into one key. That is the
|
||||
right default for this format: it has no account column, so the repeated row is almost
|
||||
always one event seen on two accounts (the fixture's two `SPLIT` rows for T at
|
||||
2026-04-16 03:00:00), and replaying it twice against the single target account would
|
||||
apply the split ratio twice.
|
||||
|
||||
The cost is the honest one: an export overlapping this one but missing some of the
|
||||
same-content rows shifts the surviving rows' ranks, so the overlap re-imports as new rows
|
||||
instead of upserting. A format with deal numbers would not have this problem; this one has
|
||||
no numbers to use.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
import io
|
||||
import json
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import (
|
||||
BrokerEvent,
|
||||
InstrumentRef,
|
||||
ParsedReport,
|
||||
ParseError,
|
||||
)
|
||||
|
||||
ZERO = Decimal(0)
|
||||
|
||||
#: Columns without which the file is not this format. Everything else is optional, so an
|
||||
#: export that drops `NKD` or `FeeCurrency` is still read.
|
||||
REQUIRED_COLUMNS = frozenset({"Event", "Date", "Symbol", "Price", "Quantity", "Currency"})
|
||||
|
||||
OPTIONAL_COLUMNS = frozenset(
|
||||
{"FeeTax", "Exchange", "NKD", "FeeCurrency", "DoNotAdjustCash", "Note"}
|
||||
)
|
||||
|
||||
#: Event types that become ledger events, and the kind each becomes.
|
||||
EVENT_KINDS: dict[str, EventKind] = {
|
||||
"BUY": EventKind.buy,
|
||||
"SELL": EventKind.sell,
|
||||
"CASH_IN": EventKind.deposit,
|
||||
"CASH_OUT": EventKind.withdrawal,
|
||||
"DIVIDEND": EventKind.dividend,
|
||||
"AMORTISATION": EventKind.amortization,
|
||||
"FEE": EventKind.commission,
|
||||
"TAX": EventKind.tax,
|
||||
"TAX_RETURN": EventKind.tax_refund,
|
||||
"SPLIT": EventKind.stock_split,
|
||||
}
|
||||
|
||||
#: Recognised event types that are deliberately NOT ledger events (see the module docstring).
|
||||
NON_LEDGER_EVENTS = frozenset({"CUSTOM_HOLDING_PRICE", "CUSTOM_HOLDING_SETTINGS"})
|
||||
|
||||
#: Substrings that mark a cash row Snowball inserted to reconcile its own replay with the
|
||||
#: broker's stated balance. Two markers, because the wording of the sentence varies with
|
||||
#: the currency and the app version but these two fragments have not.
|
||||
BALANCE_ADJUSTMENT_MARKERS = ("корректировки баланса", "не соответствует балансу")
|
||||
|
||||
#: Quotes inside the JSON blob of a CUSTOM_HOLDING_SETTINGS note.
|
||||
JSON_QUOTE_ESCAPE = "@*@"
|
||||
|
||||
#: `Exchange` value marking a paper the exchange does not quote.
|
||||
CUSTOM_HOLDING = "CUSTOM_HOLDING"
|
||||
|
||||
_TIMESTAMP_FORMATS = ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d")
|
||||
|
||||
#: 12 characters, two letters, a check digit — validated properly in `_looks_like_isin`.
|
||||
_ISIN_SHAPE = re.compile(r"^[A-Z]{2}[A-Z0-9]{9}[0-9]$")
|
||||
|
||||
#: OFZ are addressed by a MOEX secid that happens to satisfy the ISIN checksum
|
||||
#: (SU26212RMFS9). Calling it an ISIN would send the resolver looking for a security that
|
||||
#: does not exist under that number, so the shape is excluded explicitly.
|
||||
_OFZ_SECID = re.compile(r"^SU\d{5}RMFS\d$")
|
||||
|
||||
#: `Snowball_Export_<portfolio>_<dd.mm.yyyy>.csv` — the only place the portfolio is named.
|
||||
_FILENAME = re.compile(r"^.*?Export[_\s]+(?P<name>.+?)[_\s]*\d{2}\.\d{2}\.\d{4}$")
|
||||
|
||||
#: Decorations around a portfolio name in a filename (Snowball allows an emoji prefix).
|
||||
_NAME_EDGE = re.compile(r"^[^\w]+|[^\w]+$", re.UNICODE)
|
||||
|
||||
#: Whitespace that shows up inside exported numbers: plain, non-breaking, narrow.
|
||||
_SPACES = str.maketrans({" ": "", "\xa0": "", " ": ""})
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Row:
|
||||
"""One CSV data row, decoded but not yet interpreted."""
|
||||
|
||||
line_no: int
|
||||
event: str
|
||||
ts: datetime
|
||||
symbol: str
|
||||
price: Decimal | None
|
||||
quantity: Decimal | None
|
||||
currency: str
|
||||
fee_tax: Decimal | None
|
||||
exchange: str
|
||||
nkd: Decimal | None
|
||||
fee_currency: str
|
||||
note: str
|
||||
|
||||
@property
|
||||
def trade_date(self) -> date:
|
||||
return self.ts.date()
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Draft:
|
||||
"""A `BrokerEvent` with `seq` still unknown — it needs the whole file to be assigned."""
|
||||
|
||||
event: BrokerEvent
|
||||
time_key: tuple[str, str]
|
||||
"""(time of day, note) — what tells two same-day, same-content rows apart."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Accumulator:
|
||||
"""Everything the row handlers write into, so they stay free of parser plumbing."""
|
||||
|
||||
drafts: list[_Draft] = field(default_factory=list)
|
||||
instruments: dict[str, InstrumentRef] = field(default_factory=dict)
|
||||
manual_prices: list[dict[str, Any]] = field(default_factory=list)
|
||||
balance_adjustments: list[dict[str, Any]] = field(default_factory=list)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
unknown_events: dict[str, int] = field(default_factory=lambda: defaultdict(int))
|
||||
|
||||
def add_instrument(self, ref: InstrumentRef | None) -> InstrumentRef | None:
|
||||
"""Keep one ref per identity, preferring the richest description seen.
|
||||
|
||||
The same paper appears on 30 trade rows with a bare ticker and once on a settings
|
||||
row with its name, currency and sector; the ledger wants the latter.
|
||||
"""
|
||||
if ref is None:
|
||||
return None
|
||||
key = ref.key()
|
||||
known = self.instruments.get(key)
|
||||
if known is None:
|
||||
self.instruments[key] = ref
|
||||
return ref
|
||||
merged = replace(
|
||||
known,
|
||||
isin=known.isin or ref.isin,
|
||||
ticker=known.ticker or ref.ticker,
|
||||
name=known.name or ref.name,
|
||||
currency=known.currency or ref.currency,
|
||||
asset_class_hint=known.asset_class_hint or ref.asset_class_hint,
|
||||
meta={**ref.meta, **known.meta},
|
||||
)
|
||||
self.instruments[key] = merged
|
||||
return merged
|
||||
|
||||
|
||||
class UniversalCsvParser:
|
||||
"""`ReportParser` for the generic event CSV. Pure: bytes in, `ParsedReport` out."""
|
||||
|
||||
broker = "csv"
|
||||
name = "csv"
|
||||
formats: tuple[str, ...] = ("csv",)
|
||||
version = "1"
|
||||
|
||||
def sniff(self, data: bytes, filename: str) -> bool:
|
||||
"""Recognise the format by its header columns alone.
|
||||
|
||||
Not by the filename: the user renames exports, and a second tool may write the same
|
||||
columns. Not by content either — the header is the only thing every export of this
|
||||
format shares. Never raises, per the protocol.
|
||||
"""
|
||||
del filename # deliberately unused — that is the point of this method
|
||||
try:
|
||||
header = _header(data)
|
||||
except Exception:
|
||||
return False
|
||||
return header is not None and header >= REQUIRED_COLUMNS
|
||||
|
||||
def parse(self, data: bytes, filename: str) -> ParsedReport:
|
||||
"""Read the whole file, or raise `ParseError` if it is not readable as this format."""
|
||||
text = _decode(data)
|
||||
reader = csv.DictReader(io.StringIO(text))
|
||||
fieldnames = {(name or "").strip() for name in (reader.fieldnames or ())}
|
||||
if not fieldnames >= REQUIRED_COLUMNS:
|
||||
missing = ", ".join(sorted(REQUIRED_COLUMNS - fieldnames))
|
||||
raise ParseError(f"CSV без обязательных колонок: {missing}")
|
||||
|
||||
acc = _Accumulator()
|
||||
rows = _read_rows(reader, acc)
|
||||
if not rows:
|
||||
raise ParseError("CSV не содержит ни одной строки событий")
|
||||
|
||||
for row in rows:
|
||||
_dispatch(row, acc)
|
||||
|
||||
events = _assign_seq(acc.drafts)
|
||||
portfolio = _portfolio_name(filename)
|
||||
report = ParsedReport(
|
||||
broker=self.broker,
|
||||
account_external_id=portfolio,
|
||||
period_from=min(r.trade_date for r in rows),
|
||||
period_to=max(r.trade_date for r in rows),
|
||||
parser_version=self.version,
|
||||
events=events,
|
||||
positions_end=[],
|
||||
cash_end=[],
|
||||
instruments=list(acc.instruments.values()),
|
||||
warnings=acc.warnings,
|
||||
meta={
|
||||
"portfolio_name": portfolio,
|
||||
"manual_prices": acc.manual_prices,
|
||||
"balance_adjustments": acc.balance_adjustments,
|
||||
"row_count": len(rows),
|
||||
},
|
||||
)
|
||||
_finalize_warnings(report, acc)
|
||||
return report
|
||||
|
||||
|
||||
# --- reading --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _decode(data: bytes) -> str:
|
||||
"""UTF-8 with an optional BOM, falling back to the cp1251 a Russian export may carry."""
|
||||
for encoding in ("utf-8-sig", "cp1251"):
|
||||
try:
|
||||
return data.decode(encoding)
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
raise ParseError("не удалось определить кодировку CSV")
|
||||
|
||||
|
||||
def _header(data: bytes) -> frozenset[str] | None:
|
||||
"""The header's column names, read from the first line only (sniffing must be cheap)."""
|
||||
head = data[:4096].decode("utf-8-sig", errors="replace")
|
||||
line = head.splitlines()[0] if head.splitlines() else ""
|
||||
if not line:
|
||||
return None
|
||||
names = next(csv.reader([line]), [])
|
||||
return frozenset(name.strip() for name in names if name.strip())
|
||||
|
||||
|
||||
def _read_rows(reader: csv.DictReader[str], acc: _Accumulator) -> list[_Row]:
|
||||
"""Decode every data row; a row with an unreadable date is dropped with a warning.
|
||||
|
||||
The date is the one field with no sane default — it decides the period, the sort order
|
||||
and every dedupe key — so a row without it cannot be salvaged.
|
||||
"""
|
||||
rows: list[_Row] = []
|
||||
for raw in reader:
|
||||
line_no = reader.line_num
|
||||
event = _text(raw.get("Event")).upper()
|
||||
if not event:
|
||||
continue
|
||||
ts = _timestamp(_text(raw.get("Date")))
|
||||
if ts is None:
|
||||
acc.warnings.append(
|
||||
f"строка {line_no}: непонятная дата {_text(raw.get('Date'))!r}, строка пропущена"
|
||||
)
|
||||
continue
|
||||
try:
|
||||
rows.append(
|
||||
_Row(
|
||||
line_no=line_no,
|
||||
event=event,
|
||||
ts=ts,
|
||||
symbol=_text(raw.get("Symbol")),
|
||||
price=_decimal(raw.get("Price")),
|
||||
quantity=_decimal(raw.get("Quantity")),
|
||||
currency=_text(raw.get("Currency")).upper() or "RUB",
|
||||
fee_tax=_decimal(raw.get("FeeTax")),
|
||||
exchange=_text(raw.get("Exchange")).upper(),
|
||||
nkd=_decimal(raw.get("NKD")),
|
||||
fee_currency=_text(raw.get("FeeCurrency")).upper(),
|
||||
note=_text(raw.get("Note")),
|
||||
)
|
||||
)
|
||||
except InvalidOperation:
|
||||
acc.warnings.append(f"строка {line_no}: нечисловое значение в числовой колонке")
|
||||
return rows
|
||||
|
||||
|
||||
def _text(value: str | None) -> str:
|
||||
return (value or "").strip()
|
||||
|
||||
|
||||
def _decimal(value: str | None) -> Decimal | None:
|
||||
"""Decimal or None — never 0.
|
||||
|
||||
An empty cell and a zero are different facts: `FeeTax` of 0 on a trade means the broker
|
||||
charged nothing, an empty `NKD` means the row is about a share and accrued interest is
|
||||
not a concept there. Collapsing them would put a 0 ₽ НКД on every equity trade.
|
||||
"""
|
||||
raw = _text(value).translate(_SPACES).replace(",", ".")
|
||||
if not raw:
|
||||
return None
|
||||
return Decimal(raw)
|
||||
|
||||
|
||||
def _timestamp(value: str) -> datetime | None:
|
||||
for fmt in _TIMESTAMP_FORMATS:
|
||||
try:
|
||||
return datetime.strptime(value, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
# --- row handlers ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def _dispatch(row: _Row, acc: _Accumulator) -> None:
|
||||
if row.event in ("BUY", "SELL"):
|
||||
_trade(row, acc)
|
||||
elif row.event in ("CASH_IN", "CASH_OUT"):
|
||||
_cash(row, acc)
|
||||
elif row.event in ("DIVIDEND", "AMORTISATION"):
|
||||
_payout(row, acc)
|
||||
elif row.event in ("FEE", "TAX", "TAX_RETURN"):
|
||||
_charge(row, acc)
|
||||
elif row.event == "SPLIT":
|
||||
_split(row, acc)
|
||||
elif row.event == "CUSTOM_HOLDING_PRICE":
|
||||
_manual_price(row, acc)
|
||||
elif row.event == "CUSTOM_HOLDING_SETTINGS":
|
||||
_custom_holding(row, acc)
|
||||
else:
|
||||
# neither a ledger event nor a recognised non-ledger one: make it visible
|
||||
assert row.event not in EVENT_KINDS and row.event not in NON_LEDGER_EVENTS
|
||||
acc.unknown_events[row.event] += 1
|
||||
|
||||
|
||||
def _trade(row: _Row, acc: _Accumulator) -> None:
|
||||
"""BUY/SELL: `Quantity` is units, `Price` is money per unit, `NKD` is the whole НКД.
|
||||
|
||||
Accrued interest is paid on top of the price by the buyer and received by the seller,
|
||||
and the fee always leaves the account — hence the asymmetric signs below. Both are
|
||||
already inside `amount`, which is what `base.py` promises ingest.
|
||||
"""
|
||||
qty = row.quantity or ZERO
|
||||
price = row.price or ZERO
|
||||
nkd = row.nkd or ZERO
|
||||
fee = row.fee_tax
|
||||
gross = qty * price + nkd
|
||||
is_buy = row.event == "BUY"
|
||||
amount = -(gross + (fee or ZERO)) if is_buy else gross - (fee or ZERO)
|
||||
instrument = acc.add_instrument(_instrument(row))
|
||||
_emit(
|
||||
acc,
|
||||
row,
|
||||
kind=EventKind.buy if is_buy else EventKind.sell,
|
||||
amount=amount,
|
||||
instrument=instrument,
|
||||
quantity=qty if is_buy else -qty,
|
||||
price=price,
|
||||
price_currency=row.currency,
|
||||
fee=fee,
|
||||
fee_currency=row.fee_currency or row.currency,
|
||||
accrued_interest=nkd if row.nkd is not None else None,
|
||||
)
|
||||
|
||||
|
||||
def _cash(row: _Row, acc: _Accumulator) -> None:
|
||||
"""CASH_IN/CASH_OUT: `Symbol` is a currency code, `Quantity` is the amount of money.
|
||||
|
||||
Snowball's own balance corrections wear the same event type and are filtered out here —
|
||||
they are bookkeeping about the export, not money the user moved.
|
||||
"""
|
||||
amount = row.quantity or ZERO
|
||||
currency = (row.symbol or row.currency).upper()
|
||||
signed = amount if row.event == "CASH_IN" else -amount
|
||||
if _is_balance_adjustment(row.note):
|
||||
acc.balance_adjustments.append(
|
||||
{
|
||||
"kind": row.event,
|
||||
"d": row.trade_date,
|
||||
"amount": signed,
|
||||
"currency": currency,
|
||||
"note": row.note,
|
||||
"line_no": row.line_no,
|
||||
}
|
||||
)
|
||||
acc.warnings.append(
|
||||
f"строка {row.line_no}: {row.event} на {signed} {currency} — это правка остатка "
|
||||
"Snowball, а не движение денег; в леджер не попадает"
|
||||
)
|
||||
return
|
||||
_emit(
|
||||
acc,
|
||||
row,
|
||||
kind=EventKind.deposit if row.event == "CASH_IN" else EventKind.withdrawal,
|
||||
amount=signed,
|
||||
currency=currency,
|
||||
)
|
||||
|
||||
|
||||
def _payout(row: _Row, acc: _Accumulator) -> None:
|
||||
"""DIVIDEND/AMORTISATION: `Quantity` is MONEY, not units, and `Price` is a filler zero.
|
||||
|
||||
Reading `Quantity` as a position here is the single most damaging mistake this format
|
||||
invites: a 131,44 ₽ coupon would become 131,44 bonds in the lot engine.
|
||||
"""
|
||||
amount = row.quantity or ZERO
|
||||
instrument = acc.add_instrument(_instrument(row))
|
||||
meta: dict[str, Any] = {}
|
||||
if row.event == "DIVIDEND" and _looks_like_bond(row.symbol):
|
||||
meta["payout_hint"] = "coupon"
|
||||
_emit(
|
||||
acc,
|
||||
row,
|
||||
kind=EVENT_KINDS[row.event],
|
||||
amount=amount,
|
||||
instrument=instrument,
|
||||
tax=row.fee_tax if row.fee_tax else None,
|
||||
tax_currency=row.currency if row.fee_tax else None,
|
||||
meta=meta,
|
||||
)
|
||||
|
||||
|
||||
def _charge(row: _Row, acc: _Accumulator) -> None:
|
||||
"""FEE/TAX/TAX_RETURN: the money is in `FeeTax`; every other numeric column is zero.
|
||||
|
||||
These rows have no instrument even when the charge logically belongs to one — the format
|
||||
does not say which — so they stay account-level, which is also how the ledger treats
|
||||
them (`commission`/`tax` move cash and never touch a position).
|
||||
"""
|
||||
value = row.fee_tax or ZERO
|
||||
is_refund = row.event == "TAX_RETURN"
|
||||
_emit(
|
||||
acc,
|
||||
row,
|
||||
kind=EVENT_KINDS[row.event],
|
||||
amount=value if is_refund else -value,
|
||||
fee=value if row.event == "FEE" else None,
|
||||
fee_currency=row.fee_currency or row.currency if row.event == "FEE" else None,
|
||||
tax=value if row.event != "FEE" else None,
|
||||
tax_currency=row.fee_currency or row.currency if row.event != "FEE" else None,
|
||||
)
|
||||
|
||||
|
||||
def _split(row: _Row, acc: _Accumulator) -> None:
|
||||
"""SPLIT: `Price` is the ratio, and no money or quantity moves.
|
||||
|
||||
The ratio goes to `meta` under both keys `corporate_actions._stated_ratio` looks for, so
|
||||
the existing derivation picks it up as a *stated* ratio and never has to guess one from
|
||||
a transfer pair.
|
||||
"""
|
||||
ratio = row.price
|
||||
if ratio is None or ratio <= ZERO:
|
||||
acc.warnings.append(f"строка {row.line_no}: SPLIT без коэффициента, строка пропущена")
|
||||
return
|
||||
instrument = acc.add_instrument(_instrument(row))
|
||||
_emit(
|
||||
acc,
|
||||
row,
|
||||
kind=EventKind.stock_split,
|
||||
amount=ZERO,
|
||||
instrument=instrument,
|
||||
meta={"split_ratio": ratio, "ratio": ratio},
|
||||
)
|
||||
|
||||
|
||||
def _manual_price(row: _Row, acc: _Accumulator) -> None:
|
||||
"""CUSTOM_HOLDING_PRICE: a quote for `price_manual`, not an event (module docstring)."""
|
||||
if row.price is None:
|
||||
acc.warnings.append(f"строка {row.line_no}: CUSTOM_HOLDING_PRICE без цены, пропущена")
|
||||
return
|
||||
instrument = acc.add_instrument(_instrument(row))
|
||||
if instrument is None:
|
||||
acc.warnings.append(f"строка {row.line_no}: CUSTOM_HOLDING_PRICE без инструмента")
|
||||
return
|
||||
acc.manual_prices.append(
|
||||
{
|
||||
"instrument_key": instrument.key(),
|
||||
"d": row.trade_date,
|
||||
"price": row.price,
|
||||
"currency": row.currency,
|
||||
}
|
||||
)
|
||||
|
||||
|
||||
def _custom_holding(row: _Row, acc: _Accumulator) -> None:
|
||||
"""CUSTOM_HOLDING_SETTINGS: the instrument card of a user-defined holding.
|
||||
|
||||
`Note` is JSON whose quotes were rewritten to `@*@` so the CSV needs no escaping, and
|
||||
whose non-ASCII text is `\\uXXXX`-escaped — `json.loads` undoes both. A row we cannot
|
||||
read costs only the paper's human name, so it is a warning and the bare ticker survives.
|
||||
"""
|
||||
settings = _settings_json(row.note)
|
||||
if settings is None:
|
||||
acc.warnings.append(
|
||||
f"строка {row.line_no}: не удалось разобрать JSON в CUSTOM_HOLDING_SETTINGS "
|
||||
f"для {row.symbol or '?'}; инструмент остаётся без названия"
|
||||
)
|
||||
acc.add_instrument(_instrument(row))
|
||||
return
|
||||
holding = settings.get("Holding") or {}
|
||||
ref = _instrument(row)
|
||||
if ref is None:
|
||||
acc.warnings.append(f"строка {row.line_no}: CUSTOM_HOLDING_SETTINGS без символа")
|
||||
return
|
||||
acc.add_instrument(
|
||||
replace(
|
||||
ref,
|
||||
name=_text(holding.get("Description")) or ref.name,
|
||||
currency=_text(holding.get("Currency")).upper() or ref.currency,
|
||||
meta={
|
||||
**ref.meta,
|
||||
"custom_holding": True,
|
||||
"sector": _text(holding.get("Sector")) or None,
|
||||
"settings": settings.get("Settings") or {},
|
||||
},
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _settings_json(note: str) -> dict[str, Any] | None:
|
||||
if not note:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(note.replace(JSON_QUOTE_ESCAPE, '"'))
|
||||
except (ValueError, TypeError):
|
||||
return None
|
||||
return parsed if isinstance(parsed, dict) else None
|
||||
|
||||
|
||||
# --- helpers --------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _emit(
|
||||
acc: _Accumulator,
|
||||
row: _Row,
|
||||
*,
|
||||
kind: EventKind,
|
||||
amount: Decimal,
|
||||
currency: str | None = None,
|
||||
instrument: InstrumentRef | None = None,
|
||||
quantity: Decimal | None = None,
|
||||
price: Decimal | None = None,
|
||||
price_currency: str | None = None,
|
||||
fee: Decimal | None = None,
|
||||
fee_currency: str | None = None,
|
||||
tax: Decimal | None = None,
|
||||
tax_currency: str | None = None,
|
||||
accrued_interest: Decimal | None = None,
|
||||
meta: dict[str, Any] | None = None,
|
||||
) -> None:
|
||||
extra = dict(meta or {})
|
||||
extra["csv_event"] = row.event
|
||||
if row.exchange:
|
||||
extra["exchange"] = row.exchange
|
||||
acc.drafts.append(
|
||||
_Draft(
|
||||
event=BrokerEvent(
|
||||
kind=kind,
|
||||
trade_date=row.trade_date,
|
||||
amount=amount,
|
||||
currency=currency or row.currency,
|
||||
instrument=instrument,
|
||||
quantity=quantity,
|
||||
price=price,
|
||||
price_currency=price_currency,
|
||||
fee=fee,
|
||||
fee_currency=fee_currency,
|
||||
tax=tax,
|
||||
tax_currency=tax_currency,
|
||||
accrued_interest=accrued_interest,
|
||||
description=row.note or None,
|
||||
raw_line_no=row.line_no,
|
||||
meta=extra,
|
||||
),
|
||||
time_key=(row.ts.strftime("%H:%M:%S"), row.note),
|
||||
)
|
||||
)
|
||||
|
||||
|
||||
def _assign_seq(drafts: list[_Draft]) -> list[BrokerEvent]:
|
||||
"""Give each event the `seq` its fingerprint key needs (module docstring, «Dedup»).
|
||||
|
||||
Two passes on purpose: `seq` is a property of the *file*, not of the row, so it cannot
|
||||
be known while the row is being read.
|
||||
"""
|
||||
ranks: dict[tuple[str, ...], list[tuple[str, str]]] = defaultdict(list)
|
||||
for draft in drafts:
|
||||
keys = ranks[_content_key(draft.event)]
|
||||
if draft.time_key not in keys:
|
||||
keys.append(draft.time_key)
|
||||
for keys in ranks.values():
|
||||
keys.sort()
|
||||
|
||||
return [
|
||||
replace(
|
||||
draft.event,
|
||||
seq=ranks[_content_key(draft.event)].index(draft.time_key),
|
||||
)
|
||||
for draft in drafts
|
||||
]
|
||||
|
||||
|
||||
def _content_key(event: BrokerEvent) -> tuple[str, ...]:
|
||||
"""Exactly what `fingerprint_key` hashes, minus `seq` — the thing `seq` has to break."""
|
||||
return (
|
||||
event.kind.value,
|
||||
event.instrument.key() if event.instrument else "",
|
||||
event.trade_date.isoformat(),
|
||||
_norm(event.quantity),
|
||||
_norm(event.price),
|
||||
event.currency,
|
||||
_norm(event.amount),
|
||||
)
|
||||
|
||||
|
||||
def _norm(value: Decimal | None) -> str:
|
||||
if value is None:
|
||||
return ""
|
||||
normalized = value.normalize()
|
||||
return "0" if normalized == 0 else format(normalized, "f")
|
||||
|
||||
|
||||
def _instrument(row: _Row) -> InstrumentRef | None:
|
||||
"""An `InstrumentRef` from a `Symbol` that may be an ISIN, a ticker or nothing at all."""
|
||||
symbol = row.symbol
|
||||
if not symbol:
|
||||
return None
|
||||
meta: dict[str, Any] = {}
|
||||
if row.exchange:
|
||||
meta["exchange"] = row.exchange
|
||||
is_bond = _looks_like_bond(symbol)
|
||||
if _looks_like_isin(symbol):
|
||||
return InstrumentRef(
|
||||
isin=symbol,
|
||||
currency=row.currency,
|
||||
asset_class_hint="bond" if is_bond else None,
|
||||
meta=meta,
|
||||
)
|
||||
return InstrumentRef(
|
||||
ticker=symbol,
|
||||
currency=row.currency,
|
||||
asset_class_hint="bond" if is_bond else None,
|
||||
meta=meta,
|
||||
)
|
||||
|
||||
|
||||
def _looks_like_isin(symbol: str) -> bool:
|
||||
"""A 12-character code with a valid ISIN check digit, excluding OFZ secids.
|
||||
|
||||
The checksum matters: `SIBN6P4` and `SBER` fail the shape, but a shape test alone would
|
||||
happily call `SU26212RMFS9` an ISIN — and that one even passes the checksum, which is
|
||||
why the OFZ secid pattern is excluded by name rather than by arithmetic.
|
||||
"""
|
||||
if not _ISIN_SHAPE.match(symbol) or _OFZ_SECID.match(symbol):
|
||||
return False
|
||||
digits = "".join(str(int(ch, 36)) for ch in symbol)
|
||||
total = 0
|
||||
for i, ch in enumerate(reversed(digits)):
|
||||
n = int(ch)
|
||||
if i % 2 == 1:
|
||||
n *= 2
|
||||
if n > 9:
|
||||
n -= 9
|
||||
total += n
|
||||
return total % 10 == 0
|
||||
|
||||
|
||||
def _looks_like_bond(symbol: str) -> bool:
|
||||
"""Bond-shaped symbol: a real ISIN, or the OFZ secid MOEX and brokers both print.
|
||||
|
||||
Deliberately narrow. It only feeds a *hint*: a false positive would tell ingest to look
|
||||
for a coupon where a dividend was paid, and ingest would believe it until the instrument
|
||||
resolves. Papers with neither shape (a custom holding like `SIBN6P4`, which really is a
|
||||
bond) simply get no hint and are classified from the resolved instrument instead.
|
||||
"""
|
||||
return bool(_OFZ_SECID.match(symbol)) or _looks_like_isin(symbol)
|
||||
|
||||
|
||||
def _is_balance_adjustment(note: str) -> bool:
|
||||
lowered = note.lower()
|
||||
return any(marker in lowered for marker in BALANCE_ADJUSTMENT_MARKERS)
|
||||
|
||||
|
||||
def _portfolio_name(filename: str) -> str:
|
||||
"""The portfolio the export belongs to, dug out of `Snowball_Export_<name>_<date>.csv`.
|
||||
|
||||
It is not an account number and must never be mistaken for one — see the warning in
|
||||
`_finalize_warnings`. When the filename says nothing, its stem is still a better label
|
||||
than an empty string, because the user sees it in the import preview.
|
||||
"""
|
||||
stem = filename.rsplit("/", 1)[-1]
|
||||
if "." in stem:
|
||||
stem = stem.rsplit(".", 1)[0]
|
||||
match = _FILENAME.match(stem)
|
||||
raw = match.group("name") if match else stem
|
||||
return _NAME_EDGE.sub("", raw.strip()) or stem
|
||||
|
||||
|
||||
def _finalize_warnings(report: ParsedReport, acc: _Accumulator) -> None:
|
||||
"""Everything the importer must be told before it commits this file."""
|
||||
for event_type, count in sorted(acc.unknown_events.items()):
|
||||
report.warnings.append(f"неизвестный тип события {event_type!r}: пропущено строк — {count}")
|
||||
report.meta["unknown_events"] = dict(acc.unknown_events)
|
||||
report.warnings.append(
|
||||
"выгрузка не содержит номера счёта: события покрывают все брокерские счета сразу, "
|
||||
f"целевой счёт обязан указать пользователь (портфель из имени файла — "
|
||||
f"{report.account_external_id!r}); у счёта один primary_event_source, поэтому "
|
||||
"почти все события лягут со status = shadow и годятся как сверка, а не как леджер"
|
||||
)
|
||||
report.warnings.append(
|
||||
"формат не содержит закрывающих позиций и остатков денег — сверка positions_end / "
|
||||
"cash_end по этой выгрузке невозможна"
|
||||
)
|
||||
if acc.balance_adjustments:
|
||||
report.warnings.append(
|
||||
f"правок остатка, исключённых из леджера: {len(acc.balance_adjustments)} "
|
||||
"(см. meta['balance_adjustments'])"
|
||||
)
|
||||
if acc.manual_prices:
|
||||
report.warnings.append(
|
||||
f"ручных цен для price_manual: {len(acc.manual_prices)} (см. meta['manual_prices'])"
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"BALANCE_ADJUSTMENT_MARKERS",
|
||||
"EVENT_KINDS",
|
||||
"NON_LEDGER_EVENTS",
|
||||
"REQUIRED_COLUMNS",
|
||||
"UniversalCsvParser",
|
||||
]
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Which parsers exist and which one claims a given upload (plan §2: «список в registry.py»).
|
||||
|
||||
Order matters: `pick` returns the first parser whose `sniff` accepts the bytes, so a narrow
|
||||
format (a broker's own HTML) must precede a broad one (any CSV). Entry points would buy
|
||||
nothing here — the set of parsers is known at build time.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fintracker.sources.reports.base import ReportParser
|
||||
from fintracker.sources.reports.csv_universal import UniversalCsvParser
|
||||
from fintracker.sources.reports.sber import SberHtmlParser
|
||||
from fintracker.sources.reports.vtb import VtbXlsxParser
|
||||
|
||||
#: Narrow formats first, broad ones last: the universal CSV recognises any file with its
|
||||
#: column set, so it must not get a chance to claim a broker's own export.
|
||||
PARSERS: list[ReportParser] = [
|
||||
SberHtmlParser(),
|
||||
VtbXlsxParser(),
|
||||
UniversalCsvParser(),
|
||||
]
|
||||
|
||||
|
||||
def register(parser: ReportParser) -> ReportParser:
|
||||
PARSERS.append(parser)
|
||||
return parser
|
||||
|
||||
|
||||
def pick(data: bytes, filename: str) -> ReportParser | None:
|
||||
"""The first parser that recognises the file, or None when no format matches."""
|
||||
for parser in PARSERS:
|
||||
try:
|
||||
if parser.sniff(data, filename):
|
||||
return parser
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
|
||||
def get(name: str) -> ReportParser:
|
||||
for parser in PARSERS:
|
||||
if parser.name == name:
|
||||
return parser
|
||||
raise KeyError(name)
|
||||
|
||||
|
||||
def names() -> list[str]:
|
||||
return [p.name for p in PARSERS]
|
||||
@@ -0,0 +1,11 @@
|
||||
"""Sber broker report parsers.
|
||||
|
||||
Sber hands out the same report as HTML and as xlsx; the HTML one is what the user actually
|
||||
downloads, so it is what exists here. A future xlsx parser sits next to it and shares
|
||||
nothing but the `ReportParser` protocol — the two layouts differ enough that a common base
|
||||
would be a fiction.
|
||||
"""
|
||||
|
||||
from fintracker.sources.reports.sber.html import SberHtmlParser, event_dedupe_key
|
||||
|
||||
__all__ = ["SberHtmlParser", "event_dedupe_key"]
|
||||
@@ -0,0 +1,812 @@
|
||||
"""Sber's «Отчет брокера» in HTML → `ParsedReport` (plan §2, фаза 3).
|
||||
|
||||
The file is a printout, not an export: nine `<table>` elements, each introduced by a bold
|
||||
`<p>`, with two-level headers, footnote markers glued into header cells, and «Итого» rows
|
||||
mixed in with the data. So the parser keys off the section *title* and off the *cell count*
|
||||
of a row, never off a column index taken from the header — the header of the portfolio
|
||||
section alone spans three rows and eighteen columns.
|
||||
|
||||
Four decisions are worth stating, because each of them is a choice between two readings of
|
||||
the same numbers and the wrong one silently doubles money in the ledger.
|
||||
|
||||
**1. Commission is capitalised into the trade, never emitted on its own.** Sber prints the
|
||||
same fee twice: as «Комиссия Брокера»/«Комиссия Биржи» columns on the trade row, and again
|
||||
as a dated «Комиссия Брокера от DD.MM.YYYY» line in the cash movement section (aggregated
|
||||
per settlement day — 22.04's 0.60 and 1.27 arrive as one 1.87 line). Both renderings are
|
||||
verified to agree on the fixtures: the trade columns total 138.03 / 19.54, and the cash
|
||||
lines total exactly the same. Only the trade columns are kept, as `BrokerEvent.fee`, which
|
||||
`ledger/lots.py` capitalises into the lot cost; emitting the cash lines as well would put
|
||||
that same money into the ledger a second time as a separate `commission` outflow. The
|
||||
parser re-runs this arithmetic on every file and warns when the two sides disagree, because
|
||||
a disagreement means one of the two carries a fee the other does not, and the choice would
|
||||
then have to be revisited.
|
||||
|
||||
**2. «Сделка от DD.MM.YYYY» lines are dropped.** They are the T+1 cash settlement of a
|
||||
trade already recorded from the trades section, netted per day (the 23.04 line nets a sale
|
||||
and a purchase). Emitting them would double every trade's cash effect.
|
||||
|
||||
**3. «Перевод д/с с договора X» becomes a `deposit` carrying
|
||||
`meta["internal_transfer_from"]`.** Economically it *is* a deposit for this account, and
|
||||
dropping it would break the cash reconciliation — Sber's own «Пополнение счета» summary
|
||||
counts it. But the money came from another agreement of the same broker, which the user may
|
||||
later add as its own account; at that point the same rubles would be a withdrawal there and
|
||||
a deposit here, and XIRR would see one external flow where there is none. The counterparty
|
||||
number is therefore preserved on the event so `ledger/matching.py` can pair the two legs
|
||||
instead of guessing from amounts and dates.
|
||||
|
||||
**4. «Информация о зачислениях денежных средств на ИИС» is read but never emitted.** That
|
||||
table is cumulative *for the calendar year*, not for the report period: the August file
|
||||
lists top-ups back to February. Feeding it to the ledger would inflate every account whose
|
||||
history is imported from more than one file.
|
||||
|
||||
Two further notes on identity. Trades carry Sber's own deal number, stable across
|
||||
re-exports, so they dedupe on `trade_dedupe_key`. Cash rows have no number, so their
|
||||
`seq` is their index among the rows of the same `(kind, trade_date, amount)` *within the
|
||||
report* — not their line number, which differs between a full-history file and a monthly
|
||||
one describing the same operation.
|
||||
|
||||
What the format does not contain at all, on either fixture: dividends, taxes, and any
|
||||
coupon section (only footnote ⁸ mentions coupon income). A «Купонный доход» table is
|
||||
handled generically if one ever appears, and its absence is recorded in `warnings` rather
|
||||
than assumed away.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from collections import Counter
|
||||
from dataclasses import dataclass, field, replace
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
import lxml.html
|
||||
from lxml.html import HtmlElement
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import (
|
||||
BrokerEvent,
|
||||
CashEnd,
|
||||
InstrumentRef,
|
||||
ParsedReport,
|
||||
ParseError,
|
||||
PositionEnd,
|
||||
fingerprint_key,
|
||||
trade_dedupe_key,
|
||||
)
|
||||
|
||||
BROKER = "sber"
|
||||
SOURCE = "report_sber"
|
||||
|
||||
#: Whitespace Sber uses as a thousands separator, plus the ordinary space.
|
||||
_SPACES = dict.fromkeys(map(ord, " "), None)
|
||||
|
||||
_PERIOD_RE = re.compile(r"за период с\s+(\d{2}\.\d{2}\.\d{4})\s+по\s+(\d{2}\.\d{2}\.\d{4})")
|
||||
#: «Договор на ведение индивидуального инвестиционного счета S930W42 от 11.02.2026» — the
|
||||
#: number is simply the token before «от DD.MM.YYYY», which also fits «Договор … № 12345 от».
|
||||
_AGREEMENT_RE = re.compile(r"Договор[^\n]{0,200}?(\S+)\s+от\s+(\d{2}\.\d{2}\.\d{4})")
|
||||
_TRADING_CODE_RE = re.compile(r"Торговый код:\s*(\S+)")
|
||||
_TRANSFER_FROM_RE = re.compile(r"Перевод д/с с договора\s+([^\s,]+)")
|
||||
_TRANSFER_TO_RE = re.compile(r"Перевод д/с[^,]{0,80}на договор\s+([^\s,]+)")
|
||||
|
||||
#: Section titles, longest-first so «Движение денежных средств за период» is not mistaken
|
||||
#: for «Денежные средства».
|
||||
_SECTIONS: tuple[tuple[str, str], ...] = (
|
||||
("Оценка активов", "assets"),
|
||||
("Сводная информация по движению денежных средств", "summary"),
|
||||
("Информация о зачислениях денежных средств на ИИС", "iis"),
|
||||
("Портфель Ценных Бумаг", "portfolio"),
|
||||
("Движение денежных средств за период", "cash_flow"),
|
||||
("Денежные средства", "cash"),
|
||||
("Сделки купли/продажи ценных бумаг", "trades"),
|
||||
("Справочник Ценных Бумаг", "securities"),
|
||||
)
|
||||
|
||||
#: «Вид, Категория, Тип» of the securities directory → `InstrumentRef.asset_class_hint`.
|
||||
#: Checked as substrings in this order: «Биржевой фонд» must win over «фонд» inside a name.
|
||||
_ASSET_CLASS_HINTS: tuple[tuple[str, str], ...] = (
|
||||
("биржевой фонд", "etf"),
|
||||
("инвестиционный пай", "fund"),
|
||||
("пай", "fund"),
|
||||
("облигаци", "bond"),
|
||||
("акци", "share"),
|
||||
("депозитарная расписка", "share"),
|
||||
)
|
||||
|
||||
|
||||
def _text(el: HtmlElement) -> str:
|
||||
return " ".join(el.text_content().split())
|
||||
|
||||
|
||||
def _decimal(raw: str) -> Decimal | None:
|
||||
"""A Sber number, or None for an empty cell. Never float — not even transiently."""
|
||||
cleaned = raw.translate(_SPACES).replace(",", ".").lstrip("+")
|
||||
if not cleaned or cleaned == "-":
|
||||
return None
|
||||
try:
|
||||
return Decimal(cleaned)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def _date(raw: str) -> date | None:
|
||||
try:
|
||||
return datetime.strptime(raw.strip(), "%d.%m.%Y").date()
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _is_numbering(cells: list[str]) -> bool:
|
||||
"""The «1 2 3 …» row Sber prints under every header to number the columns."""
|
||||
return len(cells) > 1 and all(c.isdigit() for c in cells)
|
||||
|
||||
|
||||
def _is_total(cells: list[str]) -> bool:
|
||||
return bool(cells) and cells[0].startswith("Итого")
|
||||
|
||||
|
||||
@dataclass
|
||||
class _Directory:
|
||||
"""The securities reference, indexed for enrichment of trades and positions."""
|
||||
|
||||
refs: list[InstrumentRef] = field(default_factory=list)
|
||||
by_ticker: dict[str, InstrumentRef] = field(default_factory=dict)
|
||||
by_isin: dict[str, InstrumentRef] = field(default_factory=dict)
|
||||
by_name: dict[str, InstrumentRef] = field(default_factory=dict)
|
||||
|
||||
def add(self, ref: InstrumentRef) -> None:
|
||||
self.refs.append(ref)
|
||||
if ref.ticker:
|
||||
self.by_ticker.setdefault(ref.ticker, ref)
|
||||
if ref.isin:
|
||||
self.by_isin.setdefault(ref.isin, ref)
|
||||
if ref.name:
|
||||
self.by_name.setdefault(ref.name, ref)
|
||||
|
||||
|
||||
class SberHtmlParser:
|
||||
"""Reads one «Отчет брокера» HTML file. Pure: bytes in, `ParsedReport` out."""
|
||||
|
||||
broker = BROKER
|
||||
name = SOURCE
|
||||
formats: tuple[str, ...] = ("html", "htm")
|
||||
version = "1"
|
||||
|
||||
def sniff(self, data: bytes, filename: str) -> bool:
|
||||
"""True for a Sber broker report. Content-based — the filename is a hash for VTB
|
||||
and a template name for Sber, so neither is evidence of anything."""
|
||||
try:
|
||||
head = data[:1_000_000].decode("utf-8", errors="ignore")
|
||||
except Exception: # pragma: no cover - decode with errors="ignore" does not raise
|
||||
return False
|
||||
if "<html" not in head.lower():
|
||||
return False
|
||||
if "Отчет брокера" not in head:
|
||||
return False
|
||||
return any(
|
||||
marker in head
|
||||
for marker in ("Сделки купли/продажи ценных бумаг", "Портфель Ценных Бумаг")
|
||||
)
|
||||
|
||||
def parse(self, data: bytes, filename: str) -> ParsedReport:
|
||||
return _Parse(data, filename).run()
|
||||
|
||||
|
||||
parser = SberHtmlParser()
|
||||
|
||||
|
||||
def event_dedupe_key(report: ParsedReport, event: BrokerEvent) -> str:
|
||||
"""The plan's §1.6 A key for one parsed row.
|
||||
|
||||
Lives next to the parser because only the parser knows which of its rows carry a deal
|
||||
number: a trade keys on Sber's number and nothing else, while a cash row has to fall
|
||||
back on its economic fingerprint plus the `seq` the parser assigned.
|
||||
"""
|
||||
if event.trade_no:
|
||||
return trade_dedupe_key(report.broker, report.account_external_id, event.trade_no)
|
||||
return fingerprint_key(
|
||||
report.broker,
|
||||
report.account_external_id,
|
||||
event.kind,
|
||||
event.instrument.key() if event.instrument else "",
|
||||
event.trade_date,
|
||||
event.quantity,
|
||||
event.price,
|
||||
event.currency,
|
||||
amount=event.amount,
|
||||
seq=event.seq,
|
||||
)
|
||||
|
||||
|
||||
class _Parse:
|
||||
"""One file's parse. A short-lived object so the row counter and the warning list do
|
||||
not have to be threaded through a dozen functions."""
|
||||
|
||||
def __init__(self, data: bytes, filename: str) -> None:
|
||||
self.data = data
|
||||
self.filename = filename
|
||||
self.warnings: list[str] = []
|
||||
self.meta: dict[str, Any] = {"filename": filename}
|
||||
self.line_no = 0
|
||||
self.directory = _Directory()
|
||||
self.events: list[BrokerEvent] = []
|
||||
self.positions: list[PositionEnd] = []
|
||||
self.cash: list[CashEnd] = []
|
||||
self.summary: dict[str, Decimal] = {}
|
||||
#: Fee lines of the cash section, kept only to check them against the trade columns.
|
||||
self.cash_fees = Decimal(0)
|
||||
self.trade_fees = Decimal(0)
|
||||
|
||||
# --- driver -----------------------------------------------------------------
|
||||
|
||||
def run(self) -> ParsedReport:
|
||||
root = self._root()
|
||||
body = root.body if root.body is not None else root
|
||||
text = _text(body)
|
||||
|
||||
period_from, period_to = self._period(text)
|
||||
account = self._account(text)
|
||||
|
||||
sections = self._sections(body)
|
||||
# The securities directory is the last section but the first one needed: trades name
|
||||
# a paper by ticker only, positions by ISIN only, and both want the asset class.
|
||||
for title, kind, table in sections:
|
||||
if kind == "securities":
|
||||
self._parse_securities(title, table)
|
||||
|
||||
for title, kind, table in sections:
|
||||
handler = {
|
||||
"summary": self._parse_summary,
|
||||
"iis": self._parse_iis,
|
||||
"portfolio": self._parse_portfolio,
|
||||
"cash": self._parse_cash,
|
||||
"cash_flow": self._parse_cash_flow,
|
||||
"trades": self._parse_trades,
|
||||
"coupons": self._parse_coupons,
|
||||
# "securities" is deliberately absent: it was already read above, and
|
||||
# reading it twice would duplicate every instrument reference.
|
||||
}.get(kind)
|
||||
if handler is not None:
|
||||
handler(title, table)
|
||||
|
||||
self._check_fees()
|
||||
self._check_deposits()
|
||||
if not any(kind == "coupons" for _, kind, _ in sections):
|
||||
self.warnings.append(
|
||||
"секции «Купонный доход» в отчёте нет — купонные выплаты могли прийти только "
|
||||
"строкой движения денежных средств"
|
||||
)
|
||||
|
||||
report = ParsedReport(
|
||||
broker=BROKER,
|
||||
account_external_id=account,
|
||||
period_from=period_from,
|
||||
period_to=period_to,
|
||||
parser_version=SberHtmlParser.version,
|
||||
events=self.events,
|
||||
positions_end=self.positions,
|
||||
cash_end=self.cash,
|
||||
instruments=list(self.directory.refs),
|
||||
warnings=self.warnings,
|
||||
meta=self.meta,
|
||||
)
|
||||
return report
|
||||
|
||||
def _root(self) -> HtmlElement:
|
||||
if not self.data.strip():
|
||||
raise ParseError(f"{self.filename}: пустой файл")
|
||||
try:
|
||||
root = lxml.html.fromstring(self.data)
|
||||
except Exception as exc: # lxml raises several unrelated types on garbage input
|
||||
raise ParseError(f"{self.filename}: не разбирается как HTML: {exc}") from exc
|
||||
if root is None:
|
||||
raise ParseError(f"{self.filename}: не разбирается как HTML")
|
||||
return root
|
||||
|
||||
def _period(self, text: str) -> tuple[date, date]:
|
||||
found = _PERIOD_RE.search(text)
|
||||
if not found:
|
||||
raise ParseError(f"{self.filename}: не найден период «за период с … по …»")
|
||||
start, end = _date(found.group(1)), _date(found.group(2))
|
||||
if start is None or end is None:
|
||||
raise ParseError(f"{self.filename}: период не разбирается: {found.group(0)}")
|
||||
return start, end
|
||||
|
||||
def _account(self, text: str) -> str:
|
||||
found = _AGREEMENT_RE.search(text)
|
||||
if found:
|
||||
self.meta["agreement_opened_at"] = found.group(2)
|
||||
opened = _date(found.group(2))
|
||||
if opened is not None:
|
||||
self.meta["opened_at"] = opened.isoformat()
|
||||
return found.group(1).lstrip("№").strip()
|
||||
code = _TRADING_CODE_RE.search(text)
|
||||
if code:
|
||||
self.warnings.append("номер договора не найден, взят «Торговый код»")
|
||||
return code.group(1)
|
||||
raise ParseError(f"{self.filename}: не найден номер договора")
|
||||
|
||||
def _sections(self, body: HtmlElement) -> list[tuple[str, str, HtmlElement]]:
|
||||
"""Pair each table with the bold `<p>` that introduces it.
|
||||
|
||||
A title is consumed by the first table after it, so the unlabelled signature table
|
||||
at the end does not get parsed as a second securities directory.
|
||||
"""
|
||||
found: list[tuple[str, str, HtmlElement]] = []
|
||||
pending: str | None = None
|
||||
seen: list[HtmlElement] = []
|
||||
for el in body.iter("p", "table"):
|
||||
if el.tag == "p":
|
||||
title = _text(el)
|
||||
if title:
|
||||
pending = title
|
||||
continue
|
||||
if any(el in table.iterdescendants("table") for table in seen):
|
||||
continue # a nested table is part of its parent's rows
|
||||
seen.append(el)
|
||||
title, pending = pending, None
|
||||
if title is None:
|
||||
if "Подпись" not in _text(el):
|
||||
self.warnings.append("таблица без заголовка пропущена")
|
||||
continue
|
||||
kind = self._section_kind(title)
|
||||
if kind is None:
|
||||
self.warnings.append(f"неизвестная секция: {title[:120]}")
|
||||
continue
|
||||
found.append((title, kind, el))
|
||||
return found
|
||||
|
||||
@staticmethod
|
||||
def _section_kind(title: str) -> str | None:
|
||||
if "Купонный доход" in title:
|
||||
return "coupons"
|
||||
for prefix, kind in _SECTIONS:
|
||||
if title.startswith(prefix):
|
||||
return kind
|
||||
return None
|
||||
|
||||
# --- row plumbing -----------------------------------------------------------
|
||||
|
||||
def _rows(self, table: HtmlElement, width: int, section: str) -> list[list[str]]:
|
||||
"""Data rows of `width` cells, with headers, numbering, «Площадка:» and «Итого»
|
||||
filtered out. Anything else of an unexpected shape becomes a warning, never a
|
||||
silent skip — a column added by Sber must be visible, not absorbed."""
|
||||
rows: list[list[str]] = []
|
||||
for tr in table.iter("tr"):
|
||||
cells = [_text(td) for td in tr.iterchildren("td", "th")]
|
||||
if not cells or not any(cells):
|
||||
continue
|
||||
if _is_numbering(cells) or _is_total(cells):
|
||||
continue
|
||||
if len(cells) == 1:
|
||||
continue # «Площадка: Фондовый рынок» and other spanning captions
|
||||
if len(cells) != width and not any(any(ch.isdigit() for ch in c) for c in cells):
|
||||
continue # an upper header row of a two-level header: «Начало периода» …
|
||||
if len(cells) != width:
|
||||
self.warnings.append(
|
||||
f"{section}: строка из {len(cells)} ячеек вместо {width}: {cells[:3]}"
|
||||
)
|
||||
continue
|
||||
rows.append(cells)
|
||||
return rows
|
||||
|
||||
@staticmethod
|
||||
def _drop_header(rows: list[list[str]], first_cell: str) -> list[list[str]]:
|
||||
return [r for r in rows if r[0] != first_cell]
|
||||
|
||||
def _next_line(self) -> int:
|
||||
self.line_no += 1
|
||||
return self.line_no
|
||||
|
||||
def _total_row(self, table: HtmlElement) -> list[str] | None:
|
||||
for tr in table.iter("tr"):
|
||||
cells = [_text(td) for td in tr.iterchildren("td", "th")]
|
||||
if _is_total(cells):
|
||||
return cells
|
||||
return None
|
||||
|
||||
# --- sections ---------------------------------------------------------------
|
||||
|
||||
def _parse_summary(self, title: str, table: HtmlElement) -> None:
|
||||
"""«Сводная информация …»: Описание | Сумма | Валюта — self-check material only."""
|
||||
for cells in self._drop_header(self._rows(table, 3, title), "Описание"):
|
||||
value = _decimal(cells[1])
|
||||
if value is not None:
|
||||
self.summary[cells[0]] = value
|
||||
self.meta["summary"] = {k: str(v) for k, v in self.summary.items()}
|
||||
|
||||
def _parse_iis(self, title: str, table: HtmlElement) -> None:
|
||||
"""Deliberately not emitted: the table is cumulative for the calendar year, so the
|
||||
August file repeats February's top-ups and importing both files would double them."""
|
||||
rows = self._drop_header(self._rows(table, 6, title), "Год")
|
||||
self.meta["iis_contributions_ignored"] = len(rows)
|
||||
|
||||
def _parse_portfolio(self, title: str, table: HtmlElement) -> None:
|
||||
"""«Портфель Ценных Бумаг» → `positions_end`.
|
||||
|
||||
18 columns: 0 name, 1 ISIN, 2 currency, 3-7 opening, 8-12 closing, 13-14 change,
|
||||
15-17 planned. Closing is what reconciliation compares against, so only 8-12 are read.
|
||||
"""
|
||||
for cells in self._drop_header(self._rows(table, 18, title), "Наименование"):
|
||||
qty = _decimal(cells[8])
|
||||
if qty is None:
|
||||
self.warnings.append(f"{title}: позиция без количества: {cells[0]}")
|
||||
continue
|
||||
self._next_line()
|
||||
isin = cells[1] or None
|
||||
known = self.directory.by_isin.get(isin or "") or self.directory.by_name.get(cells[0])
|
||||
self.positions.append(
|
||||
PositionEnd(
|
||||
instrument=InstrumentRef(
|
||||
isin=isin,
|
||||
ticker=known.ticker if known else None,
|
||||
name=cells[0] or None,
|
||||
currency=cells[2] or None,
|
||||
asset_class_hint=known.asset_class_hint if known else None,
|
||||
source_key=f"ISIN:{isin}" if isin else f"NAME:{cells[0]}",
|
||||
),
|
||||
qty=qty,
|
||||
price=_decimal(cells[10]),
|
||||
market_value=_decimal(cells[11]),
|
||||
currency=cells[2] or None,
|
||||
accrued_interest=_decimal(cells[12]),
|
||||
)
|
||||
)
|
||||
|
||||
def _parse_cash(self, title: str, table: HtmlElement) -> None:
|
||||
"""«Денежные средства» → `cash_end`: column 5 is the closing balance per currency."""
|
||||
for cells in self._drop_header(self._rows(table, 9, title), "Торговая площадка"):
|
||||
balance = _decimal(cells[5])
|
||||
if balance is None:
|
||||
self.warnings.append(f"{title}: остаток не разобран: {cells[:3]}")
|
||||
continue
|
||||
self._next_line()
|
||||
self.cash.append(CashEnd(currency=cells[1], balance=balance))
|
||||
|
||||
def _parse_cash_flow(self, title: str, table: HtmlElement) -> None:
|
||||
"""«Движение денежных средств за период» → deposits, withdrawals and payouts.
|
||||
|
||||
Trade settlements and fee lines are dropped here (see the module docstring); what
|
||||
remains is money crossing the account boundary, which is what XIRR is computed on.
|
||||
"""
|
||||
rows = self._drop_header(self._rows(table, 6, title), "Дата")
|
||||
credits_total = Decimal(0)
|
||||
debits_total = Decimal(0)
|
||||
staged: list[BrokerEvent] = []
|
||||
for cells in rows:
|
||||
when = _date(cells[0])
|
||||
if when is None:
|
||||
self.warnings.append(f"{title}: дата не разобрана: {cells[:3]}")
|
||||
continue
|
||||
self._next_line()
|
||||
description = cells[2]
|
||||
currency = cells[3] or "RUB"
|
||||
credit = _decimal(cells[4]) or Decimal(0)
|
||||
debit = _decimal(cells[5]) or Decimal(0)
|
||||
credits_total += credit
|
||||
debits_total += debit
|
||||
amount = credit - debit
|
||||
|
||||
if description.startswith("Сделка от"):
|
||||
continue # settlement of a trade already taken from the trades section
|
||||
if description.startswith(("Комиссия Брокера", "Комиссия Биржи")):
|
||||
self.cash_fees += debit - credit
|
||||
continue # capitalised into the trade's `fee`
|
||||
|
||||
kind, meta = self._classify_cash(description, amount)
|
||||
if kind is None:
|
||||
self.warnings.append(f"{title}: неизвестная операция «{description}»")
|
||||
kind = EventKind.other
|
||||
meta["section"] = title
|
||||
staged.append(
|
||||
BrokerEvent(
|
||||
kind=kind,
|
||||
trade_date=when,
|
||||
amount=amount,
|
||||
currency=currency,
|
||||
description=description,
|
||||
raw_line_no=self.line_no,
|
||||
meta=meta,
|
||||
)
|
||||
)
|
||||
|
||||
self._check_cash_flow_total(title, table, credits_total, debits_total)
|
||||
self.events.extend(self._numbered(staged))
|
||||
|
||||
@staticmethod
|
||||
def _classify_cash(
|
||||
description: str, amount: Decimal
|
||||
) -> tuple[EventKind | None, dict[str, Any]]:
|
||||
"""Description → kind. Sber writes free text, so this matches on the stable head of
|
||||
each phrase and refuses to guess: an unrecognised row becomes `other` plus a warning
|
||||
rather than a deposit inferred from its sign."""
|
||||
meta: dict[str, Any] = {}
|
||||
lowered = description.lower()
|
||||
transfer_from = _TRANSFER_FROM_RE.search(description)
|
||||
if transfer_from:
|
||||
# Another agreement of the same broker. Kept as a deposit because it really does
|
||||
# fund this account, but tagged so the other leg can be recognised later and the
|
||||
# pair not counted as two external flows.
|
||||
meta["internal_transfer_from"] = transfer_from.group(1)
|
||||
return (EventKind.deposit if amount >= 0 else EventKind.withdrawal), meta
|
||||
transfer_to = _TRANSFER_TO_RE.search(description)
|
||||
if transfer_to:
|
||||
meta["internal_transfer_to"] = transfer_to.group(1)
|
||||
return (EventKind.withdrawal if amount <= 0 else EventKind.deposit), meta
|
||||
if "купон" in lowered:
|
||||
return EventKind.coupon, meta
|
||||
if "дивиденд" in lowered:
|
||||
return EventKind.dividend, meta
|
||||
if "налог" in lowered:
|
||||
return (EventKind.tax if amount <= 0 else EventKind.tax_refund), meta
|
||||
if lowered.startswith("зачисление д/с"):
|
||||
return EventKind.deposit, meta
|
||||
if lowered.startswith(("вывод д/с", "списание д/с", "снятие д/с")):
|
||||
return EventKind.withdrawal, meta
|
||||
return None, meta
|
||||
|
||||
def _parse_trades(self, title: str, table: HtmlElement) -> None:
|
||||
"""«Сделки купли/продажи ценных бумаг» → buy/sell events.
|
||||
|
||||
16 columns: 0 trade date, 1 settle date, 2 time, 3 name, 4 ticker, 5 currency,
|
||||
6 side, 7 qty, 8 price, 9 amount, 10 accrued interest, 11 broker fee,
|
||||
12 exchange fee, 13 deal number, 14 comment, 15 status.
|
||||
"""
|
||||
gross = Decimal(0)
|
||||
accrued_total = Decimal(0)
|
||||
broker_total = Decimal(0)
|
||||
exchange_total = Decimal(0)
|
||||
for cells in self._drop_header(self._rows(table, 16, title), "Дата заключения"):
|
||||
trade_date = _date(cells[0])
|
||||
qty = _decimal(cells[7])
|
||||
price = _decimal(cells[8])
|
||||
amount = _decimal(cells[9])
|
||||
if trade_date is None or qty is None or amount is None:
|
||||
self.warnings.append(f"{title}: сделка не разобрана: {cells[:5]}")
|
||||
continue
|
||||
accrued = _decimal(cells[10]) or Decimal(0)
|
||||
broker_fee = _decimal(cells[11]) or Decimal(0)
|
||||
exchange_fee = _decimal(cells[12]) or Decimal(0)
|
||||
side = cells[6]
|
||||
if side.startswith("Покупка"):
|
||||
kind = EventKind.buy
|
||||
quantity = qty
|
||||
cash = -(amount + accrued + broker_fee + exchange_fee)
|
||||
elif side.startswith("Продажа"):
|
||||
kind = EventKind.sell
|
||||
quantity = -qty
|
||||
cash = amount + accrued - broker_fee - exchange_fee
|
||||
else:
|
||||
self.warnings.append(f"{title}: неизвестный вид сделки «{side}»")
|
||||
continue
|
||||
|
||||
gross += amount
|
||||
accrued_total += accrued
|
||||
broker_total += broker_fee
|
||||
exchange_total += exchange_fee
|
||||
self._next_line()
|
||||
|
||||
currency = cells[5] or "RUB"
|
||||
known = self.directory.by_ticker.get(cells[4]) or self.directory.by_name.get(cells[3])
|
||||
instrument = InstrumentRef(
|
||||
isin=known.isin if known else None,
|
||||
ticker=cells[4] or (known.ticker if known else None),
|
||||
name=cells[3] or None,
|
||||
currency=currency,
|
||||
asset_class_hint=known.asset_class_hint if known else None,
|
||||
source_key=(f"ISIN:{known.isin}" if known and known.isin else f"TICKER:{cells[4]}"),
|
||||
)
|
||||
meta: dict[str, Any] = {"section": title, "status": cells[15] or None}
|
||||
if cells[2]:
|
||||
meta["trade_time"] = cells[2]
|
||||
if cells[14]:
|
||||
meta["comment"] = cells[14]
|
||||
self.events.append(
|
||||
BrokerEvent(
|
||||
kind=kind,
|
||||
trade_date=trade_date,
|
||||
settle_date=_date(cells[1]),
|
||||
amount=cash,
|
||||
currency=currency,
|
||||
instrument=instrument,
|
||||
quantity=quantity,
|
||||
price=price,
|
||||
price_currency=currency,
|
||||
fee=broker_fee + exchange_fee,
|
||||
fee_currency=currency,
|
||||
accrued_interest=accrued or None,
|
||||
trade_no=cells[13] or None,
|
||||
description=f"{side} {cells[3]}".strip(),
|
||||
raw_line_no=self.line_no,
|
||||
meta=meta,
|
||||
)
|
||||
)
|
||||
self.trade_fees = broker_total + exchange_total
|
||||
self._check_trades_total(title, table, gross, accrued_total, broker_total, exchange_total)
|
||||
|
||||
def _parse_securities(self, title: str, table: HtmlElement) -> None:
|
||||
"""«Справочник Ценных Бумаг» → `instruments`: 0 name, 1 ticker, 2 ISIN, 3 issuer,
|
||||
4 kind/category, 5 issue."""
|
||||
for cells in self._drop_header(self._rows(table, 6, title), "Наименование"):
|
||||
isin = cells[2] or None
|
||||
ticker = cells[1] or None
|
||||
if not isin and not ticker:
|
||||
self.warnings.append(f"{title}: бумага без ISIN и тикера: {cells[0]}")
|
||||
continue
|
||||
self._next_line()
|
||||
self.directory.add(
|
||||
InstrumentRef(
|
||||
isin=isin,
|
||||
ticker=ticker,
|
||||
name=cells[0] or None,
|
||||
asset_class_hint=_asset_class_hint(cells[4]),
|
||||
source_key=f"ISIN:{isin}" if isin else f"TICKER:{ticker}",
|
||||
meta={"issuer": cells[3] or None, "issue": cells[5] or None, "kind": cells[4]},
|
||||
)
|
||||
)
|
||||
|
||||
def _parse_coupons(self, title: str, table: HtmlElement) -> None:
|
||||
"""A «Купонный доход» table, should Sber ever print one.
|
||||
|
||||
Neither fixture contains this section — only footnote ⁸ refers to coupon income —
|
||||
so the layout is unknown and the parser reads it by header name rather than by
|
||||
column position, and says so in `warnings` instead of pretending it is verified.
|
||||
"""
|
||||
rows = [
|
||||
[_text(td) for td in tr.iterchildren("td", "th")]
|
||||
for tr in table.iter("tr")
|
||||
if any(_text(td) for td in tr.iterchildren("td", "th"))
|
||||
]
|
||||
header = next((r for r in rows if any("ата" in c for c in r)), None)
|
||||
if header is None:
|
||||
self.warnings.append(f"{title}: не найден заголовок таблицы, секция пропущена")
|
||||
return
|
||||
index = {
|
||||
key: i
|
||||
for i, cell in enumerate(header)
|
||||
for key in ("date", "amount", "isin", "name", "qty", "tax")
|
||||
if _matches_column(cell, key)
|
||||
}
|
||||
if "date" not in index or "amount" not in index:
|
||||
self.warnings.append(f"{title}: нет колонок даты и суммы, секция пропущена")
|
||||
return
|
||||
staged: list[BrokerEvent] = []
|
||||
for cells in rows:
|
||||
if cells is header or _is_numbering(cells) or _is_total(cells):
|
||||
continue
|
||||
if len(cells) != len(header):
|
||||
continue
|
||||
when = _date(cells[index["date"]])
|
||||
amount = _decimal(cells[index["amount"]])
|
||||
if when is None or amount is None:
|
||||
continue
|
||||
self._next_line()
|
||||
isin = cells[index["isin"]] if "isin" in index else None
|
||||
known = self.directory.by_isin.get(isin or "")
|
||||
staged.append(
|
||||
BrokerEvent(
|
||||
kind=EventKind.coupon,
|
||||
trade_date=when,
|
||||
amount=amount,
|
||||
currency="RUB",
|
||||
instrument=known
|
||||
or (InstrumentRef(isin=isin, source_key=f"ISIN:{isin}") if isin else None),
|
||||
quantity=_decimal(cells[index["qty"]]) if "qty" in index else None,
|
||||
tax=_decimal(cells[index["tax"]]) if "tax" in index else None,
|
||||
description=cells[index["name"]] if "name" in index else None,
|
||||
raw_line_no=self.line_no,
|
||||
meta={"section": title},
|
||||
)
|
||||
)
|
||||
self.warnings.append(
|
||||
f"{title}: секция разобрана эвристикой по названиям колонок, фикстуры для неё нет"
|
||||
)
|
||||
self.events.extend(self._numbered(staged))
|
||||
|
||||
# --- self-checks ------------------------------------------------------------
|
||||
|
||||
@staticmethod
|
||||
def _numbered(events: list[BrokerEvent]) -> list[BrokerEvent]:
|
||||
"""Assign `seq` within `(kind, trade_date, amount)` groups.
|
||||
|
||||
The point is the overlapping-period case: the same 10 551.18 ₽ transfer of 13.08
|
||||
is row 24 of the full-history file and row 1 of the August file. Numbering within
|
||||
the group makes it 0 in both, so the fingerprint key matches and the second import
|
||||
upserts instead of duplicating.
|
||||
"""
|
||||
counter: Counter[tuple[str, date, Decimal]] = Counter()
|
||||
out: list[BrokerEvent] = []
|
||||
for event in events:
|
||||
group = (event.kind.value, event.trade_date, event.amount)
|
||||
seq = counter[group]
|
||||
counter[group] += 1
|
||||
out.append(replace(event, seq=seq))
|
||||
return out
|
||||
|
||||
def _check_trades_total(
|
||||
self,
|
||||
title: str,
|
||||
table: HtmlElement,
|
||||
gross: Decimal,
|
||||
accrued: Decimal,
|
||||
broker_fee: Decimal,
|
||||
exchange_fee: Decimal,
|
||||
) -> None:
|
||||
"""«Итого, RUB | сумма | НКД | комиссия брокера | комиссия биржи»."""
|
||||
total = self._total_row(table)
|
||||
if total is None or len(total) < 5:
|
||||
self.warnings.append(f"{title}: итоговая строка не найдена")
|
||||
return
|
||||
for label, stated, computed in (
|
||||
("сумма сделок", _decimal(total[1]), gross),
|
||||
("НКД", _decimal(total[2]), accrued),
|
||||
("комиссия брокера", _decimal(total[3]), broker_fee),
|
||||
("комиссия биржи", _decimal(total[4]), exchange_fee),
|
||||
):
|
||||
if stated is not None and stated != computed:
|
||||
self.warnings.append(
|
||||
f"{title}: {label} не сходится с итогом: {computed} против {stated}"
|
||||
)
|
||||
|
||||
def _check_cash_flow_total(
|
||||
self, title: str, table: HtmlElement, credits: Decimal, debits: Decimal
|
||||
) -> None:
|
||||
total = self._total_row(table)
|
||||
if total is None or len(total) < 3:
|
||||
self.warnings.append(f"{title}: итоговая строка не найдена")
|
||||
return
|
||||
stated_credit, stated_debit = _decimal(total[1]), _decimal(total[2])
|
||||
if stated_credit is not None and stated_credit != credits:
|
||||
self.warnings.append(
|
||||
f"{title}: зачисления не сходятся с итогом: {credits} против {stated_credit}"
|
||||
)
|
||||
if stated_debit is not None and stated_debit != debits:
|
||||
self.warnings.append(
|
||||
f"{title}: списания не сходятся с итогом: {debits} против {stated_debit}"
|
||||
)
|
||||
|
||||
def _check_fees(self) -> None:
|
||||
"""The whole basis for capitalising fees: both renderings must state the same money.
|
||||
|
||||
If they ever diverge, one of them carries a fee the other does not and dropping the
|
||||
cash lines would lose it — so the divergence has to surface, not be assumed away.
|
||||
"""
|
||||
if self.cash_fees != self.trade_fees:
|
||||
self.warnings.append(
|
||||
"комиссии расходятся: в сделках "
|
||||
f"{self.trade_fees}, отдельными строками движения денег {self.cash_fees}; "
|
||||
"в события попали только комиссии из сделок"
|
||||
)
|
||||
|
||||
def _check_deposits(self) -> None:
|
||||
"""«Пополнение счета» of the summary counts transfers from the broker's other
|
||||
agreements alongside ordinary top-ups, so both kinds are summed here."""
|
||||
stated = self.summary.get("Пополнение счета")
|
||||
if stated is None:
|
||||
return
|
||||
parsed = sum(
|
||||
(e.amount for e in self.events if e.kind == EventKind.deposit),
|
||||
start=Decimal(0),
|
||||
)
|
||||
if parsed != stated:
|
||||
self.warnings.append(f"пополнения не сходятся со сводкой: {parsed} против {stated}")
|
||||
|
||||
|
||||
def _asset_class_hint(kind: str) -> str | None:
|
||||
lowered = kind.lower()
|
||||
for marker, hint in _ASSET_CLASS_HINTS:
|
||||
if marker in lowered:
|
||||
return hint
|
||||
return None
|
||||
|
||||
|
||||
def _matches_column(cell: str, key: str) -> bool:
|
||||
lowered = cell.lower()
|
||||
return {
|
||||
"date": "дата" in lowered,
|
||||
"amount": "сумма" in lowered and "налог" not in lowered,
|
||||
"isin": "isin" in lowered,
|
||||
"name": "наименование" in lowered,
|
||||
"qty": "количество" in lowered,
|
||||
"tax": "налог" in lowered,
|
||||
}[key]
|
||||
@@ -0,0 +1,10 @@
|
||||
"""VTB brokerage report parsers.
|
||||
|
||||
`common.py` holds the format-independent half (what a row means, how a key is built),
|
||||
`xlsx.py` the container-specific half. A pdf rendering of the same report would be a third
|
||||
module reusing `common.py` — the contract it has to honour is written down there.
|
||||
"""
|
||||
|
||||
from fintracker.sources.reports.vtb.xlsx import VtbXlsxParser
|
||||
|
||||
__all__ = ["VtbXlsxParser"]
|
||||
@@ -0,0 +1,584 @@
|
||||
"""Everything in a VTB report that does not depend on the container it arrived in.
|
||||
|
||||
VTB renders one and the same back-office report as xlsx and as pdf. The wording of every
|
||||
section, the vocabulary of its columns and the sign conventions are identical in both; only
|
||||
the way one gets at a value differs — a cell address versus a position on a page. So the
|
||||
split here is deliberate: `xlsx.py` is allowed to know about worksheets and merged cells and
|
||||
nothing else, while this module owns the meaning — how «EQMX ETF, 3965, RU000A101EJ5» turns
|
||||
into an `InstrumentRef`, what «Вид сделки: Продажа» does to the sign of a quantity, and how a
|
||||
row becomes a `BrokerEvent` in `event`'s conventions.
|
||||
|
||||
**The contract a future `vtb/pdf.py` must satisfy.** Two files describing the same month
|
||||
must produce the same `dedupe_key` for the same deal, otherwise a re-import doubles the
|
||||
ledger. The key of a trade is `sha1(broker | account_external_id | trade_no)`, so a pdf
|
||||
parser has to extract exactly three things identically:
|
||||
|
||||
* `broker` — the constant `BROKER` here, never derived from the file;
|
||||
* `account_external_id` — the **agreement number** («№ и дата Соглашения»), the same string
|
||||
in both renderings, and not the personal account number («Лицевой счет»), which is
|
||||
personal data we never carry;
|
||||
* `trade_no` — column «№ сделки» (`Z` in xlsx, e.g. `B17476399721`), **not** «№ сделки у
|
||||
организатора торгов» (`AC`, `17476399721`). The key is already scoped by broker and
|
||||
account, so exchange-wide uniqueness buys nothing; what matters is that VTB prints the
|
||||
same «№ сделки» in every rendering of its own report, while the organiser's number is
|
||||
absent for over-the-counter deals in some formats and is the exchange's identifier, not
|
||||
VTB's.
|
||||
|
||||
Cash rows carry no deal number and fall back to `fingerprint_key`, which needs the date, the
|
||||
signed amount, the currency and a stable `seq` — for a pdf parser `seq` must be the row's
|
||||
index **inside its own section**, counted the same way as here, not a page line number.
|
||||
|
||||
Everything else a pdf parser extracts (the venue, the order number, the counterparty) lands
|
||||
in `meta` and never touches a key, so the two formats may legitimately differ there.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import (
|
||||
BrokerEvent,
|
||||
InstrumentRef,
|
||||
fingerprint_key,
|
||||
trade_dedupe_key,
|
||||
)
|
||||
|
||||
BROKER = "vtb"
|
||||
"""`Broker` value and the first component of every dedupe key this parser builds."""
|
||||
|
||||
PARSER_NAME = "report_vtb"
|
||||
|
||||
#: VTB prints the pre-ISO Soviet code for the rouble everywhere — in balances, in the price
|
||||
#: currency of a position and in the settlement currency of a deal. The project stores
|
||||
#: three-letter ISO 4217 codes (`String(3)` columns, `fx_rate_daily.ccy`), so the alias is
|
||||
#: applied at the parser boundary and `RUR` never reaches the ledger.
|
||||
CURRENCY_ALIASES = {"RUR": "RUB"}
|
||||
|
||||
#: «Вид сделки» / «Тип операции» of a securities row -> ledger kind. The report states the
|
||||
#: direction in words and keeps the quantity positive; the sign is ours to apply.
|
||||
TRADE_SIDES: dict[str, EventKind] = {
|
||||
"покупка": EventKind.buy,
|
||||
"продажа": EventKind.sell,
|
||||
}
|
||||
|
||||
#: «Тип операции» in «Движение денежных средств» -> ledger kind, or None for a row that must
|
||||
#: NOT become an event. «Сальдо расчетов по сделкам с ценными бумагами» is exactly that: it
|
||||
#: is the money leg of deals already emitted from the trades table, netted per settlement
|
||||
#: day, and emitting it would count every purchase twice — once as `buy.amount`, once as a
|
||||
#: cash movement. «Вознаграждение брокера» is also None here, but for a different reason:
|
||||
#: the fee is capitalised into the deal that caused it (see `broker_fee_residual`).
|
||||
CASH_OPERATIONS: dict[str, EventKind | None] = {
|
||||
"зачисление денежных средств": EventKind.deposit,
|
||||
"списание денежных средств": EventKind.withdrawal,
|
||||
"вывод денежных средств": EventKind.withdrawal,
|
||||
"перевод денежных средств": EventKind.transfer_in,
|
||||
"налог на доходы физических лиц": EventKind.tax,
|
||||
"дивиденды": EventKind.dividend,
|
||||
"купонный доход": EventKind.coupon,
|
||||
"сальдо расчетов по сделкам с ценными бумагами": None,
|
||||
"вознаграждение брокера": None,
|
||||
"вознаграждение депозитария": None,
|
||||
}
|
||||
|
||||
#: Group headers inside «Отчёт об остатках ценных бумаг» -> `InstrumentRef.asset_class_hint`.
|
||||
#: A hint only: the user confirms the class when a pending instrument is resolved.
|
||||
ASSET_CLASS_HINTS: dict[str, str] = {
|
||||
"пай": "etf",
|
||||
"акция": "share",
|
||||
"адр": "share",
|
||||
"облигация": "bond",
|
||||
"еврооблигация": "bond",
|
||||
"депозитарная расписка": "share",
|
||||
}
|
||||
|
||||
#: Sub-headers and market names that sit in the first column of a data section and are not
|
||||
#: rows: skipping them silently is correct, warning about them would be noise.
|
||||
SECTION_SUBHEADERS = frozenset(
|
||||
{
|
||||
"основной рынок",
|
||||
"срочный рынок",
|
||||
"внебирж. рынок",
|
||||
"внебиржевой рынок",
|
||||
"итого",
|
||||
"итого:",
|
||||
}
|
||||
)
|
||||
|
||||
_PERIOD_RE = re.compile(
|
||||
r"за период с\s+(\d{2}\.\d{2}\.\d{4})\s+по\s+(\d{2}\.\d{2}\.\d{4})", re.IGNORECASE
|
||||
)
|
||||
_ISIN_RE = re.compile(r"\b([A-Z]{2}[A-Z0-9]{9}\d)\b")
|
||||
_TICKER_RE = re.compile(r"^[A-Z][A-Z0-9]{1,11}$")
|
||||
_DATE_RE = re.compile(r"^(\d{2})\.(\d{2})\.(\d{4})")
|
||||
|
||||
#: Excel's day zero on Windows: serial 1 is 1900-01-01 and serial 60 is the mythical
|
||||
#: 1900-02-29, so counting from 1899-12-30 is right for everything after that.
|
||||
_EXCEL_EPOCH = date(1899, 12, 30)
|
||||
|
||||
|
||||
def normalize_text(value: Any) -> str:
|
||||
"""Lowercased, whitespace-collapsed, ё-folded — the form every label is matched in.
|
||||
|
||||
VTB wraps header captions with newlines to fit a column and is inconsistent about «ё»
|
||||
(«субсчёту» in one section title, «отчет» in another), so matching raw strings would
|
||||
make the parser fail on cosmetic edits to the template.
|
||||
"""
|
||||
if value is None:
|
||||
return ""
|
||||
text = str(value).replace("\xa0", " ").replace("ё", "е").replace("Ё", "Е")
|
||||
return re.sub(r"\s+", " ", text).strip().lower()
|
||||
|
||||
|
||||
def normalize_currency(value: Any) -> str | None:
|
||||
"""`RUR` -> `RUB`; anything else uppercased as printed. None for an empty cell."""
|
||||
text = str(value or "").strip().upper()
|
||||
if not text:
|
||||
return None
|
||||
return CURRENCY_ALIASES.get(text, text)
|
||||
|
||||
|
||||
def to_decimal(value: Any) -> Decimal | None:
|
||||
"""Money and quantities as `Decimal`, never through `Decimal(float)`.
|
||||
|
||||
openpyxl hands back a `float` for every numeric cell, and `Decimal(1.24)` would store
|
||||
1.2399999999999999911182158029987476766109466552734375. Going through `str()` keeps the
|
||||
number the report actually printed. Strings arrive too (VTB stores the position price as
|
||||
text) and may use a comma or a thin space, which the pdf rendering will also do.
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, Decimal):
|
||||
return value
|
||||
if isinstance(value, int):
|
||||
return Decimal(value)
|
||||
if isinstance(value, float):
|
||||
return Decimal(str(value))
|
||||
text = str(value).replace("\xa0", "").replace(" ", "").replace(",", ".").strip()
|
||||
if not text or text in {"-", "—"}:
|
||||
return None
|
||||
try:
|
||||
return Decimal(text)
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def to_date(value: Any) -> date | None:
|
||||
"""A cell that should hold a date, whatever the writer chose to put there.
|
||||
|
||||
openpyxl usually applies the number format and returns `datetime`, but a cell VTB left
|
||||
unformatted comes back as the bare serial (`46247.0`, or `46247.664…` when a time is
|
||||
attached), and a pdf will hand over `17.09.2026` as text. All three are accepted.
|
||||
"""
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, datetime):
|
||||
return value.date()
|
||||
if isinstance(value, date):
|
||||
return value
|
||||
if isinstance(value, (int, float)):
|
||||
serial = float(value)
|
||||
if serial <= 0:
|
||||
return None
|
||||
return _EXCEL_EPOCH + timedelta(days=int(serial))
|
||||
text = str(value).strip()
|
||||
match = _DATE_RE.match(text)
|
||||
if match:
|
||||
day, month, year = (int(part) for part in match.groups())
|
||||
try:
|
||||
return date(year, month, day)
|
||||
except ValueError:
|
||||
return None
|
||||
return None
|
||||
|
||||
|
||||
def to_datetime(value: Any) -> datetime | None:
|
||||
"""Same as `to_date`, but keeps the time of a deal when the cell carries one."""
|
||||
if isinstance(value, datetime):
|
||||
return value
|
||||
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
||||
serial = float(value)
|
||||
if serial <= 0:
|
||||
return None
|
||||
return datetime(_EXCEL_EPOCH.year, _EXCEL_EPOCH.month, _EXCEL_EPOCH.day) + timedelta(
|
||||
days=serial
|
||||
)
|
||||
if isinstance(value, str):
|
||||
text = value.strip()
|
||||
for fmt in ("%d.%m.%Y %H:%M:%S", "%d.%m.%Y %H:%M", "%d.%m.%Y"):
|
||||
try:
|
||||
return datetime.strptime(text, fmt)
|
||||
except ValueError:
|
||||
continue
|
||||
day = to_date(value)
|
||||
return datetime(day.year, day.month, day.day) if day else None
|
||||
|
||||
|
||||
def parse_period(text: str) -> tuple[date, date] | None:
|
||||
"""The reporting period out of the title line.
|
||||
|
||||
The period lives in the title and nowhere else: the file name is a GUID
|
||||
(`reportd0235f1e-….xlsx`), and the section rows only show the dates on which something
|
||||
happened, which for a quiet month is not the period at all.
|
||||
"""
|
||||
match = _PERIOD_RE.search(str(text or ""))
|
||||
if not match:
|
||||
return None
|
||||
first, second = (to_date(part) for part in match.groups())
|
||||
if first is None or second is None:
|
||||
return None
|
||||
return first, second
|
||||
|
||||
|
||||
def parse_security(
|
||||
title: Any,
|
||||
*,
|
||||
asset_class_hint: str | None = None,
|
||||
venue: str | None = None,
|
||||
currency: str | None = None,
|
||||
extra: dict[str, Any] | None = None,
|
||||
) -> InstrumentRef:
|
||||
"""«EQMX ETF, 3965, RU000A101EJ5» -> an `InstrumentRef`.
|
||||
|
||||
VTB glues the name, the state registration number and the ISIN into one cell with commas
|
||||
— and the name itself may contain commas («ПАО "Сбербанк России", ао»), so the fields are
|
||||
taken from the end: the last comma-separated part that looks like an ISIN is the ISIN,
|
||||
and a purely numeric or `dash`-shaped part before it is the registration number. What is
|
||||
left is the name.
|
||||
|
||||
The ticker is not printed as a field, but VTB starts an ETF's name with it («EQMX ETF»),
|
||||
so the first word is taken as a ticker candidate only when it looks like one. It is never
|
||||
the sole identity: resolution goes by ISIN first, and `source_key` is the ISIN whenever
|
||||
there is one, so a wrong guess costs nothing.
|
||||
"""
|
||||
text = str(title or "").replace("\n", " ").strip()
|
||||
parts = [part.strip() for part in text.split(",") if part.strip()]
|
||||
isin: str | None = None
|
||||
reg_number: str | None = None
|
||||
|
||||
for index in range(len(parts) - 1, -1, -1):
|
||||
match = _ISIN_RE.fullmatch(parts[index].upper())
|
||||
if match:
|
||||
isin = match.group(1)
|
||||
if index > 0 and re.fullmatch(r"[\w\-/]{1,20}", parts[index - 1]):
|
||||
reg_number = parts[index - 1]
|
||||
parts = parts[: index - 1]
|
||||
else:
|
||||
parts = parts[:index]
|
||||
break
|
||||
|
||||
name = ", ".join(parts).strip() or text or None
|
||||
ticker: str | None = None
|
||||
if name:
|
||||
first_word = name.split()[0].upper()
|
||||
if _TICKER_RE.fullmatch(first_word):
|
||||
ticker = first_word
|
||||
|
||||
meta: dict[str, Any] = {"title": text}
|
||||
if reg_number:
|
||||
meta["reg_number"] = reg_number
|
||||
if venue:
|
||||
meta["venue"] = venue
|
||||
if extra:
|
||||
meta.update({key: value for key, value in extra.items() if value is not None})
|
||||
|
||||
return InstrumentRef(
|
||||
isin=isin,
|
||||
ticker=ticker,
|
||||
name=name,
|
||||
currency=normalize_currency(currency),
|
||||
asset_class_hint=asset_class_hint,
|
||||
source_key=f"ISIN:{isin}" if isin else f"VTB:{text}",
|
||||
meta=meta,
|
||||
)
|
||||
|
||||
|
||||
def trade_side_kind(value: Any) -> EventKind | None:
|
||||
"""«Покупка»/«Продажа» -> `buy`/`sell`; None for a wording we do not know."""
|
||||
return TRADE_SIDES.get(normalize_text(value))
|
||||
|
||||
|
||||
def cash_operation_kind(value: Any) -> tuple[EventKind | None, bool]:
|
||||
"""`(kind, known)` for a «Тип операции» of the cash section.
|
||||
|
||||
Two different Nones have to be told apart: a row we deliberately drop (`known=True`,
|
||||
kind None) and a wording nobody has mapped yet (`known=False`), which must reach
|
||||
`warnings` rather than disappear.
|
||||
"""
|
||||
key = normalize_text(value)
|
||||
if key in CASH_OPERATIONS:
|
||||
return CASH_OPERATIONS[key], True
|
||||
return None, False
|
||||
|
||||
|
||||
def build_trade_event(
|
||||
*,
|
||||
security: Any,
|
||||
side: Any,
|
||||
trade_dt: Any,
|
||||
settle_date: Any,
|
||||
quantity: Any,
|
||||
price: Any,
|
||||
price_currency: Any,
|
||||
settlement_currency: Any,
|
||||
amount_gross: Any,
|
||||
accrued_interest: Any = None,
|
||||
fees: tuple[Any, ...] = (),
|
||||
trade_no: Any = None,
|
||||
asset_class_hint: str | None = None,
|
||||
venue: Any = None,
|
||||
order_no: Any = None,
|
||||
exchange_trade_no: Any = None,
|
||||
counterparty: Any = None,
|
||||
raw_line_no: int = 0,
|
||||
seq: int = 0,
|
||||
) -> BrokerEvent:
|
||||
"""One row of «Заключенные в отчетном периоде сделки» as a ledger event.
|
||||
|
||||
Two decisions are baked in here.
|
||||
|
||||
**The fee is capitalised into the deal.** VTB prints its commission twice: per deal in
|
||||
«Комиссия Банка за расчёт/за заключение сделки», and again as a «Вознаграждение брокера»
|
||||
line in the cash section (the two agree to the kopeck — the parser checks it and warns if
|
||||
they ever stop agreeing). Only one of them may become an event: `ledger/lots.py`
|
||||
capitalises `fee` into the cost of the lot, so emitting the cash line as a separate
|
||||
`commission` event on top would charge the commission twice — once inside the lot's cost
|
||||
basis, once as a standalone expense. The deal-level figure is the one kept, because it is
|
||||
the one that knows which lot it belongs to.
|
||||
|
||||
**`amount` includes the fee, `fee` stays positive.** That is `event`'s convention:
|
||||
a buy pays `-(sum + fee)`, a sale receives `+(sum - fee)`, and `fee` is stated positive
|
||||
so that summing fees never double-counts against the cash effect.
|
||||
|
||||
Raises `ValueError` when the row lacks something a trade cannot exist without — the
|
||||
caller turns that into a warning naming the line, never into a silent skip.
|
||||
"""
|
||||
kind = trade_side_kind(side)
|
||||
if kind is None:
|
||||
raise ValueError(f"неизвестный вид сделки «{side}»")
|
||||
|
||||
day = to_datetime(trade_dt)
|
||||
if day is None:
|
||||
raise ValueError("нет даты заключения сделки")
|
||||
|
||||
qty = to_decimal(quantity)
|
||||
if qty is None:
|
||||
raise ValueError("нет количества")
|
||||
qty = abs(qty)
|
||||
|
||||
gross = to_decimal(amount_gross)
|
||||
if gross is None:
|
||||
raise ValueError("нет суммы сделки")
|
||||
gross = abs(gross)
|
||||
|
||||
fee_total = sum((to_decimal(part) or Decimal(0) for part in fees), Decimal(0))
|
||||
currency = normalize_currency(settlement_currency) or normalize_currency(price_currency)
|
||||
if currency is None:
|
||||
raise ValueError("нет валюты расчётов")
|
||||
|
||||
if kind is EventKind.buy:
|
||||
quantity_signed = qty
|
||||
amount = -(gross + fee_total)
|
||||
else:
|
||||
quantity_signed = -qty
|
||||
amount = gross - fee_total
|
||||
|
||||
meta: dict[str, Any] = {"section": "trades", "ts": day.isoformat()}
|
||||
for key, value in (
|
||||
("venue", venue),
|
||||
("order_no", order_no),
|
||||
("exchange_trade_no", exchange_trade_no),
|
||||
("counterparty", counterparty),
|
||||
):
|
||||
text = str(value).strip() if value is not None else ""
|
||||
if text:
|
||||
meta[key] = text
|
||||
|
||||
return BrokerEvent(
|
||||
kind=kind,
|
||||
trade_date=day.date(),
|
||||
amount=amount,
|
||||
currency=currency,
|
||||
settle_date=to_date(settle_date),
|
||||
instrument=parse_security(
|
||||
security,
|
||||
asset_class_hint=asset_class_hint,
|
||||
venue=str(venue).strip() if venue else None,
|
||||
currency=price_currency,
|
||||
),
|
||||
quantity=quantity_signed,
|
||||
price=to_decimal(price),
|
||||
price_currency=normalize_currency(price_currency),
|
||||
fee=fee_total if fee_total else None,
|
||||
fee_currency=currency if fee_total else None,
|
||||
accrued_interest=to_decimal(accrued_interest),
|
||||
trade_no=str(trade_no).strip() if trade_no else None,
|
||||
description=f"{str(side).strip()} {qty} шт.",
|
||||
raw_line_no=raw_line_no,
|
||||
seq=seq,
|
||||
meta=meta,
|
||||
)
|
||||
|
||||
|
||||
def build_cash_event(
|
||||
*,
|
||||
kind: EventKind,
|
||||
operation: Any,
|
||||
trade_date: Any,
|
||||
amount: Any,
|
||||
currency: Any,
|
||||
comment: Any = None,
|
||||
market: str | None = None,
|
||||
raw_line_no: int = 0,
|
||||
seq: int = 0,
|
||||
) -> BrokerEvent:
|
||||
"""One row of «Движение денежных средств» as a ledger event.
|
||||
|
||||
The amount is taken with the sign VTB printed: the column is already signed by its effect
|
||||
on the account (+26 100 credited, −3.64 charged), which is `event.amount`'s convention.
|
||||
"""
|
||||
day = to_date(trade_date)
|
||||
if day is None:
|
||||
raise ValueError("нет даты операции")
|
||||
value = to_decimal(amount)
|
||||
if value is None:
|
||||
raise ValueError("нет суммы операции")
|
||||
code = normalize_currency(currency)
|
||||
if code is None:
|
||||
raise ValueError("нет валюты операции")
|
||||
|
||||
meta: dict[str, Any] = {"section": "cash", "operation": str(operation).strip()}
|
||||
if market:
|
||||
meta["market"] = market
|
||||
|
||||
text = str(comment).strip() if comment else ""
|
||||
return BrokerEvent(
|
||||
kind=kind,
|
||||
trade_date=day,
|
||||
amount=value,
|
||||
currency=code,
|
||||
fee=abs(value) if kind is EventKind.commission else None,
|
||||
fee_currency=code if kind is EventKind.commission else None,
|
||||
description=text or str(operation).strip() or None,
|
||||
raw_line_no=raw_line_no,
|
||||
seq=seq,
|
||||
meta=meta,
|
||||
)
|
||||
|
||||
|
||||
def dedupe_key(account_external_id: str, event: BrokerEvent) -> str:
|
||||
"""The key `ledger/ingest.py` will upsert on — see this module's docstring for the
|
||||
cross-format contract it encodes."""
|
||||
if event.trade_no:
|
||||
return trade_dedupe_key(BROKER, account_external_id, event.trade_no)
|
||||
instrument_key = event.instrument.key() if event.instrument else ""
|
||||
return fingerprint_key(
|
||||
BROKER,
|
||||
account_external_id,
|
||||
event.kind,
|
||||
instrument_key,
|
||||
event.trade_date,
|
||||
event.quantity,
|
||||
event.price,
|
||||
event.currency,
|
||||
amount=event.amount,
|
||||
seq=event.seq,
|
||||
)
|
||||
|
||||
|
||||
def broker_fee_residual(
|
||||
cash_fee_total: Decimal, trade_fee_total: Decimal, *, tolerance: Decimal = Decimal("0.005")
|
||||
) -> Decimal:
|
||||
"""How much of the brokerage fee the trades table does not account for.
|
||||
|
||||
Zero in every report seen so far — VTB's «Вознаграждение брокера» is the sum of the
|
||||
per-deal commissions to the kopeck. If a report ever charges a fee that belongs to no
|
||||
deal (a monthly service charge, a depository fee billed under the same wording), the
|
||||
difference is real money that would otherwise vanish from the ledger, so the caller emits
|
||||
it as a standalone `commission` event and warns.
|
||||
"""
|
||||
residual = cash_fee_total - trade_fee_total
|
||||
return residual if abs(residual) > tolerance else Decimal(0)
|
||||
|
||||
|
||||
def merge_instruments(refs: list[InstrumentRef]) -> list[InstrumentRef]:
|
||||
"""Unique instruments in first-seen order, the asset-class hint carried over.
|
||||
|
||||
A paper is named in three sections and only the positions table states its class, so the
|
||||
hint found there is applied to the same ISIN wherever else it appeared.
|
||||
"""
|
||||
hints = {ref.key(): ref.asset_class_hint for ref in refs if ref.asset_class_hint}
|
||||
merged: dict[str, InstrumentRef] = {}
|
||||
for ref in refs:
|
||||
key = ref.key()
|
||||
hint = ref.asset_class_hint or hints.get(key)
|
||||
current = merged.get(key)
|
||||
if current is None:
|
||||
merged[key] = ref if ref.asset_class_hint == hint else _with_hint(ref, hint)
|
||||
elif current.asset_class_hint is None and hint:
|
||||
merged[key] = _with_hint(current, hint)
|
||||
return list(merged.values())
|
||||
|
||||
|
||||
def _with_hint(ref: InstrumentRef, hint: str | None) -> InstrumentRef:
|
||||
if hint is None or ref.asset_class_hint == hint:
|
||||
return ref
|
||||
return InstrumentRef(
|
||||
isin=ref.isin,
|
||||
ticker=ref.ticker,
|
||||
board=ref.board,
|
||||
name=ref.name,
|
||||
currency=ref.currency,
|
||||
asset_class_hint=hint,
|
||||
source_key=ref.source_key,
|
||||
meta=dict(ref.meta),
|
||||
)
|
||||
|
||||
|
||||
def apply_instrument_hints(
|
||||
events: list[BrokerEvent], refs: list[InstrumentRef]
|
||||
) -> list[BrokerEvent]:
|
||||
"""Re-stamp events with the asset-class hint learned from the positions table.
|
||||
|
||||
The trades table does not say whether a paper is a share, a unit or a bond; the positions
|
||||
table does, in its group headers. Both describe the same ISINs, so the hint is filled in
|
||||
after the whole file is read.
|
||||
"""
|
||||
hints = {ref.key(): ref.asset_class_hint for ref in refs if ref.asset_class_hint}
|
||||
result: list[BrokerEvent] = []
|
||||
for event in events:
|
||||
instrument = event.instrument
|
||||
if instrument is None or instrument.asset_class_hint is not None:
|
||||
result.append(event)
|
||||
continue
|
||||
hint = hints.get(instrument.key())
|
||||
if hint is None:
|
||||
result.append(event)
|
||||
continue
|
||||
result.append(
|
||||
BrokerEvent(
|
||||
kind=event.kind,
|
||||
trade_date=event.trade_date,
|
||||
amount=event.amount,
|
||||
currency=event.currency,
|
||||
settle_date=event.settle_date,
|
||||
instrument=_with_hint(instrument, hint),
|
||||
quantity=event.quantity,
|
||||
price=event.price,
|
||||
price_currency=event.price_currency,
|
||||
fee=event.fee,
|
||||
fee_currency=event.fee_currency,
|
||||
tax=event.tax,
|
||||
tax_currency=event.tax_currency,
|
||||
accrued_interest=event.accrued_interest,
|
||||
trade_no=event.trade_no,
|
||||
description=event.description,
|
||||
raw_line_no=event.raw_line_no,
|
||||
seq=event.seq,
|
||||
meta=dict(event.meta),
|
||||
)
|
||||
)
|
||||
return result
|
||||
@@ -0,0 +1,704 @@
|
||||
"""The xlsx rendering of a VTB brokerage report: worksheet in, `ParsedReport` out.
|
||||
|
||||
The file is a printed page saved as a spreadsheet, not a data export. Everything lives on a
|
||||
single sheet called `brokerage_report`, sections follow one another vertically and are
|
||||
identified only by their Russian title sitting in column B, and every field is a merged
|
||||
range — which is why values land in columns as scattered as C, F, I, L, O, R, AB, AF. There
|
||||
is no dimension record either, so `reset_dimensions()` is required before iterating or
|
||||
openpyxl reports a 1x1 sheet.
|
||||
|
||||
Consequently the parser never addresses a grid by offset. It walks rows, switches on the
|
||||
section title in column B, and inside a section asks a fixed question per column letter —
|
||||
the letters being the only stable thing in the layout. A row that does not answer the
|
||||
question (no date where a date must be, no quantity where a quantity must be) ends the
|
||||
section instead of being force-fitted, and anything unrecognised goes to `warnings`.
|
||||
|
||||
What is deliberately NOT emitted as an event, because VTB states the same fact three times:
|
||||
|
||||
* «Движение ценных бумаг» is the settlement leg of the deals in the trades table — same
|
||||
papers, same quantities, one day later. It is used to cross-check quantities and nothing
|
||||
else.
|
||||
* «Завершенные в отчетном периоде сделки» repeats «Заключенные» verbatim, deal numbers
|
||||
included; for a fully settled period the two tables are identical. Emitting it would
|
||||
double every trade — and because the deal number is the same, the duplicate would not even
|
||||
be caught by the dedupe key, it would simply be the same event parsed twice. It is used to
|
||||
assert that the two sets of deal numbers agree.
|
||||
* «Сальдо расчетов по сделкам с ценными бумагами» in the cash section is the money leg of
|
||||
those same deals, netted per settlement day.
|
||||
|
||||
All of the meaning — signs, kinds, dedupe keys, the instrument string — lives in
|
||||
`common.py`, so that a future pdf parser reuses it instead of re-deriving it.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import re
|
||||
import warnings as warnings_module
|
||||
import zipfile
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from openpyxl import load_workbook
|
||||
from openpyxl.utils import column_index_from_string
|
||||
|
||||
from fintracker.models.ledger import EventKind
|
||||
from fintracker.sources.reports.base import (
|
||||
BrokerEvent,
|
||||
CashEnd,
|
||||
InstrumentRef,
|
||||
ParsedReport,
|
||||
ParseError,
|
||||
PositionEnd,
|
||||
)
|
||||
from fintracker.sources.reports.vtb import common
|
||||
from fintracker.sources.reports.vtb.common import (
|
||||
BROKER,
|
||||
normalize_currency,
|
||||
normalize_text,
|
||||
to_date,
|
||||
to_decimal,
|
||||
)
|
||||
|
||||
SHEET_NAME = "brokerage_report"
|
||||
|
||||
#: Sections this parser reads, keyed by their normalised title in column B.
|
||||
SECTIONS: dict[str, str] = {
|
||||
"сводная информация по субсчету клиента": "summary",
|
||||
"отчет об остатках денежных средств": "cash_balances",
|
||||
"движение денежных средств": "cash_flow",
|
||||
"отчет об остатках ценных бумаг": "positions",
|
||||
"движение ценных бумаг": "security_moves",
|
||||
"заключенные в отчетном периоде сделки с ценными бумагами": "trades",
|
||||
"завершенные в отчетном периоде сделки с ценными бумагами (обязательства прекращены)": (
|
||||
"trades_completed"
|
||||
),
|
||||
}
|
||||
|
||||
#: Sections VTB itself lists as possible (the boilerplate paragraph at the end of the file
|
||||
#: enumerates them) but which this account has never produced. Meeting one is not an error —
|
||||
#: it is a report with derivatives or currency trades in it — but it must be visible, so it
|
||||
#: becomes a warning rather than a silent skip.
|
||||
KNOWN_UNPARSED_SECTIONS = frozenset(
|
||||
{
|
||||
"задолженность клиента",
|
||||
"обязательства/требования по передаче дохода (заблокировано)",
|
||||
"блокировки / аресты",
|
||||
"открытые позиции по производным финансовым инструментам",
|
||||
"обязательства и требования по незавершенным сделкам",
|
||||
"заключенные в отчетном периоде сделки с иностранной валютой",
|
||||
"завершенные в отчетном периоде сделки с иностранной валютой (обязательства прекращены)",
|
||||
"незавершенные в отчетном периоде сделки с ценными бумагами (обязательства не исполнены)",
|
||||
"незавершенные в отчетном периоде сделки с иностранной валютой "
|
||||
"(обязательства не исполнены)",
|
||||
"сделки с производными финансовыми инструментами в отчетном периоде",
|
||||
"завершенные в отчетном периоде сделки по переносу открытой позиции клиента",
|
||||
}
|
||||
)
|
||||
|
||||
#: Header labels in column B whose value sits in the first non-empty cell to the right.
|
||||
_HEADER_FIELDS = {
|
||||
"№ и дата соглашения": "agreement",
|
||||
"№ субсчета:": "subaccount",
|
||||
"на ведение иис": "iis_opened_at",
|
||||
"дата формирования отчета": "generated_at",
|
||||
}
|
||||
|
||||
_SUMMARY_OPENING = "входящий остаток денежных средств"
|
||||
_SUMMARY_CLOSING = "исходящий остаток денежных средств"
|
||||
_SUMMARY_SETTLEMENT = "сальдо расчетов по сделкам с ценными бумагами"
|
||||
_SUMMARY_BROKER_FEE = "вознаграждение брокера"
|
||||
|
||||
_CURRENCY_CODE_RE = re.compile(r"^[a-z]{3}$")
|
||||
_FOOTNOTE_RE = re.compile(r"^\s*\d+\s*[-–—]\s")
|
||||
_MONEY_TOLERANCE = Decimal("0.01")
|
||||
|
||||
|
||||
def _cell(row: tuple[Any, ...], column: str) -> Any:
|
||||
index = column_index_from_string(column) - 1
|
||||
return row[index] if index < len(row) else None
|
||||
|
||||
|
||||
def _text(row: tuple[Any, ...], column: str) -> str:
|
||||
value = _cell(row, column)
|
||||
return "" if value is None else str(value).strip()
|
||||
|
||||
|
||||
def _value_right_of(row: tuple[Any, ...], column: str) -> Any:
|
||||
"""The first non-empty cell after `column` — the value of a label in a merged layout.
|
||||
|
||||
The header fields are printed as «label in B, value somewhere to the right»; in today's
|
||||
template the value is in I, but the template moves it whenever a label gets longer, and
|
||||
scanning is cheap.
|
||||
"""
|
||||
start = column_index_from_string(column)
|
||||
for value in row[start:]:
|
||||
if value is not None and str(value).strip():
|
||||
return value
|
||||
return None
|
||||
|
||||
|
||||
def _is_boilerplate(text: str) -> bool:
|
||||
"""Legal prose and footnotes that trail the report — never a section, never data."""
|
||||
return bool(_FOOTNOTE_RE.match(text)) or "\n" in text or len(text) > 200
|
||||
|
||||
|
||||
class VtbXlsxParser:
|
||||
"""`ReportParser` for VTB's xlsx brokerage report."""
|
||||
|
||||
broker = BROKER
|
||||
name = common.PARSER_NAME
|
||||
formats: tuple[str, ...] = ("xlsx",)
|
||||
version = "1"
|
||||
|
||||
def sniff(self, data: bytes, filename: str) -> bool:
|
||||
"""Recognise the file by its content only.
|
||||
|
||||
The name carries nothing: VTB calls every download `report<guid>.xlsx`. What is
|
||||
reliable is the workbook's single sheet name (`brokerage_report`, written into
|
||||
`xl/workbook.xml`) together with the bank's own title string in the shared strings
|
||||
table. Both are read straight out of the zip, so sniffing costs no parse — and never
|
||||
raises: an unreadable or foreign file is simply not ours.
|
||||
"""
|
||||
del filename # deliberately unused: it is a GUID, it says nothing about the format
|
||||
try:
|
||||
if not data.startswith(b"PK\x03\x04"):
|
||||
return False
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as archive:
|
||||
names = set(archive.namelist())
|
||||
if "xl/workbook.xml" not in names:
|
||||
return False
|
||||
workbook = archive.read("xl/workbook.xml").decode("utf-8", "replace")
|
||||
if SHEET_NAME not in workbook:
|
||||
return False
|
||||
for entry in ("xl/sharedStrings.xml", "xl/worksheets/sheet1.xml"):
|
||||
if entry not in names:
|
||||
continue
|
||||
content = archive.read(entry).decode("utf-8", "replace")
|
||||
if "Банка ВТБ" in content and "за период с" in content:
|
||||
return True
|
||||
return False
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
def parse(self, data: bytes, filename: str) -> ParsedReport:
|
||||
"""Parse the sheet. `filename` is kept only as traceability in `meta.source_file`:
|
||||
nothing in the report is derived from it."""
|
||||
rows = _load_rows(data)
|
||||
return _Parse(rows, source_file=filename).run()
|
||||
|
||||
|
||||
def _load_rows(data: bytes) -> list[tuple[Any, ...]]:
|
||||
try:
|
||||
with warnings_module.catch_warnings():
|
||||
# The file ships without a default style; openpyxl warns and copes, and the
|
||||
# warning would otherwise surface on every import the user makes.
|
||||
warnings_module.simplefilter("ignore")
|
||||
workbook = load_workbook(io.BytesIO(data), read_only=True, data_only=True)
|
||||
try:
|
||||
sheet = (
|
||||
workbook[SHEET_NAME]
|
||||
if SHEET_NAME in workbook.sheetnames
|
||||
else workbook.worksheets[0]
|
||||
)
|
||||
# No dimension record in the file: without this the sheet reports 1x1.
|
||||
sheet.reset_dimensions()
|
||||
return [tuple(row) for row in sheet.iter_rows(values_only=True)]
|
||||
finally:
|
||||
workbook.close()
|
||||
except ParseError:
|
||||
raise
|
||||
except Exception as exc: # любой сбой openpyxl/zip значит «файл нечитаем»
|
||||
raise ParseError(f"не удалось прочитать xlsx ВТБ: {exc}") from exc
|
||||
|
||||
|
||||
class _Parse:
|
||||
"""One pass over the sheet plus the cross-checks that follow it.
|
||||
|
||||
Kept as a class only because a dozen accumulators would otherwise be threaded through
|
||||
every handler by hand; there is no state that outlives `run()`.
|
||||
"""
|
||||
|
||||
def __init__(self, rows: list[tuple[Any, ...]], *, source_file: str = "") -> None:
|
||||
self.rows = rows
|
||||
self.source_file = source_file
|
||||
self.warnings: list[str] = []
|
||||
self.events: list[BrokerEvent] = []
|
||||
self.positions: list[PositionEnd] = []
|
||||
self.cash_end: list[CashEnd] = []
|
||||
self.instruments: list[InstrumentRef] = []
|
||||
self.summary: dict[str, Decimal] = {}
|
||||
self.summary_currency: str | None = None
|
||||
|
||||
self.section: str | None = None
|
||||
self.expect_caption = False
|
||||
self.started = False
|
||||
self.asset_class_hint: str | None = None
|
||||
self.market: str | None = None
|
||||
self.section_seq = 0
|
||||
|
||||
self.trade_numbers: list[str] = []
|
||||
self.completed_numbers: list[str] = []
|
||||
self.trade_gross_signed = Decimal(0)
|
||||
self.trade_fee_total = Decimal(0)
|
||||
self.trade_qty: dict[str, Decimal] = {}
|
||||
self.move_qty: dict[str, Decimal] = {}
|
||||
self.move_rows = 0
|
||||
self.cash_fee_total = Decimal(0)
|
||||
self.cash_settlement_total = Decimal(0)
|
||||
self.positions_total_rub: Decimal | None = None
|
||||
self.last_fee_row: tuple[Any, ...] | None = None
|
||||
|
||||
# -- entry point ---------------------------------------------------------------------
|
||||
|
||||
def run(self) -> ParsedReport:
|
||||
period, header = self._read_header()
|
||||
for line_no, row in enumerate(self.rows, start=1):
|
||||
self._row(line_no, row)
|
||||
|
||||
self.instruments = common.merge_instruments(self.instruments)
|
||||
self.events = common.apply_instrument_hints(self.events, self.instruments)
|
||||
self._reconcile()
|
||||
|
||||
meta: dict[str, Any] = {
|
||||
"subaccount": header.get("subaccount"),
|
||||
"iis_opened_at": header.get("iis_opened_at"),
|
||||
"generated_at": header.get("generated_at"),
|
||||
"summary": {key: str(value) for key, value in self.summary.items()},
|
||||
"source_file": self.source_file or None,
|
||||
}
|
||||
return ParsedReport(
|
||||
broker=BROKER,
|
||||
account_external_id=header["agreement"],
|
||||
period_from=period[0],
|
||||
period_to=period[1],
|
||||
parser_version=VtbXlsxParser.version,
|
||||
events=self.events,
|
||||
positions_end=self.positions,
|
||||
cash_end=self.cash_end,
|
||||
instruments=self.instruments,
|
||||
warnings=self.warnings,
|
||||
meta={key: value for key, value in meta.items() if value is not None},
|
||||
)
|
||||
|
||||
# -- header --------------------------------------------------------------------------
|
||||
|
||||
def _read_header(self) -> tuple[tuple[Any, Any], dict[str, Any]]:
|
||||
"""The period from the title line and the agreement number from the label block.
|
||||
|
||||
Both are mandatory: without a period the report cannot be placed on a timeline, and
|
||||
without the agreement number no event can be keyed, so either missing is a
|
||||
`ParseError` rather than a warning.
|
||||
"""
|
||||
period = None
|
||||
fields: dict[str, Any] = {}
|
||||
for row in self.rows:
|
||||
first = normalize_text(_cell(row, "B"))
|
||||
if first in SECTIONS:
|
||||
break
|
||||
for value in row:
|
||||
if period is None and isinstance(value, str):
|
||||
period = common.parse_period(value)
|
||||
if first in _HEADER_FIELDS:
|
||||
fields[_HEADER_FIELDS[first]] = _value_right_of(row, "B")
|
||||
|
||||
if period is None:
|
||||
raise ParseError("в отчёте ВТБ не найден период («за период с … по …»)")
|
||||
|
||||
agreement = str(fields.get("agreement") or "").strip()
|
||||
if not agreement:
|
||||
raise ParseError("в отчёте ВТБ не найден номер соглашения")
|
||||
|
||||
header: dict[str, Any] = {"agreement": agreement}
|
||||
subaccount = str(fields.get("subaccount") or "").strip()
|
||||
if subaccount:
|
||||
header["subaccount"] = subaccount
|
||||
for key in ("iis_opened_at", "generated_at"):
|
||||
day = to_date(fields.get(key))
|
||||
if day is not None:
|
||||
header[key] = day.isoformat()
|
||||
return period, header
|
||||
|
||||
# -- row dispatch --------------------------------------------------------------------
|
||||
|
||||
def _row(self, line_no: int, row: tuple[Any, ...]) -> None:
|
||||
first = _cell(row, "B")
|
||||
title = normalize_text(first)
|
||||
|
||||
if title in SECTIONS:
|
||||
self.section = SECTIONS[title]
|
||||
self.expect_caption = True
|
||||
self.asset_class_hint = None
|
||||
self.market = None
|
||||
self.section_seq = 0
|
||||
self.started = True
|
||||
return
|
||||
if not self.started:
|
||||
return
|
||||
if title in KNOWN_UNPARSED_SECTIONS:
|
||||
self.section = None
|
||||
self.warnings.append(f"строка {line_no}: секция «{first}» не разбирается парсером")
|
||||
return
|
||||
if not title:
|
||||
# A blank line, or the second line of a two-storey column caption.
|
||||
return
|
||||
if self.expect_caption:
|
||||
self.expect_caption = False
|
||||
return
|
||||
|
||||
handler = getattr(self, f"_row_{self.section}", None) if self.section else None
|
||||
if handler is not None and handler(line_no, row):
|
||||
return
|
||||
|
||||
# The row does not fit the section it follows: the section is over.
|
||||
self.section = None
|
||||
text = str(first).strip()
|
||||
if not _is_boilerplate(text):
|
||||
self.warnings.append(f"строка {line_no}: нераспознанная строка/секция «{text[:80]}»")
|
||||
|
||||
# -- sections ------------------------------------------------------------------------
|
||||
|
||||
def _row_summary(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
amount = to_decimal(_cell(row, "L"))
|
||||
if amount is None:
|
||||
return False
|
||||
currency = normalize_currency(_cell(row, "O"))
|
||||
if currency is None:
|
||||
# Every summary line is денежная; a missing currency means the layout moved and
|
||||
# the totals below are being compared across currencies without knowing it.
|
||||
self.warnings.append(
|
||||
f"строка {line_no}: строка сводки «{_text(row, 'B')[:60]}» без валюты"
|
||||
)
|
||||
self.summary[normalize_text(_cell(row, "B"))] = amount
|
||||
self.summary_currency = self.summary_currency or currency
|
||||
return True
|
||||
|
||||
def _row_cash_balances(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
label = normalize_text(_cell(row, "B"))
|
||||
if label.startswith("сумма денежных средств"):
|
||||
# The all-currency total in roubles: a derived line, not a balance of its own.
|
||||
return True
|
||||
if not _CURRENCY_CODE_RE.match(label):
|
||||
return False
|
||||
# AF is «Итого» of the closing balance across markets; R alone is the main market.
|
||||
balance = to_decimal(_cell(row, "AF"))
|
||||
if balance is None:
|
||||
balance = to_decimal(_cell(row, "R"))
|
||||
if balance is None:
|
||||
self.warnings.append(f"строка {line_no}: остаток денежных средств без суммы")
|
||||
return True
|
||||
currency = normalize_currency(_cell(row, "B"))
|
||||
assert currency is not None
|
||||
self.cash_end.append(CashEnd(currency=currency, balance=balance))
|
||||
return True
|
||||
|
||||
def _row_cash_flow(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
label = normalize_text(_cell(row, "B"))
|
||||
if label in common.SECTION_SUBHEADERS:
|
||||
self.market = str(_cell(row, "B")).strip()
|
||||
return True
|
||||
day = to_date(_cell(row, "B"))
|
||||
if day is None:
|
||||
return False
|
||||
|
||||
self.section_seq += 1
|
||||
operation = _cell(row, "J")
|
||||
amount = to_decimal(_cell(row, "C"))
|
||||
if amount is None:
|
||||
self.warnings.append(f"строка {line_no}: движение денег без суммы, строка пропущена")
|
||||
return True
|
||||
|
||||
kind, known = common.cash_operation_kind(operation)
|
||||
operation_key = normalize_text(operation)
|
||||
if operation_key == _SUMMARY_BROKER_FEE:
|
||||
self.cash_fee_total += amount
|
||||
self.last_fee_row = row
|
||||
return True
|
||||
if operation_key == _SUMMARY_SETTLEMENT:
|
||||
self.cash_settlement_total += amount
|
||||
return True
|
||||
if not known:
|
||||
self.warnings.append(
|
||||
f"строка {line_no}: неизвестный тип денежной операции «{operation}», "
|
||||
"записан как other"
|
||||
)
|
||||
kind = EventKind.other
|
||||
if kind is None:
|
||||
return True
|
||||
|
||||
try:
|
||||
self.events.append(
|
||||
common.build_cash_event(
|
||||
kind=kind,
|
||||
operation=operation,
|
||||
trade_date=_cell(row, "B"),
|
||||
amount=_cell(row, "C"),
|
||||
currency=_cell(row, "G"),
|
||||
comment=_cell(row, "P"),
|
||||
market=self.market,
|
||||
raw_line_no=line_no,
|
||||
seq=self.section_seq,
|
||||
)
|
||||
)
|
||||
except ValueError as exc:
|
||||
self.warnings.append(f"строка {line_no}: {exc}, строка пропущена")
|
||||
return True
|
||||
|
||||
def _row_positions(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
label = normalize_text(_cell(row, "B"))
|
||||
if label in common.ASSET_CLASS_HINTS:
|
||||
self.asset_class_hint = common.ASSET_CLASS_HINTS[label]
|
||||
return True
|
||||
if label in common.SECTION_SUBHEADERS:
|
||||
self.positions_total_rub = to_decimal(_cell(row, "AF"))
|
||||
return True
|
||||
|
||||
qty = to_decimal(_cell(row, "L"))
|
||||
if qty is None:
|
||||
return False
|
||||
|
||||
currency = normalize_currency(_cell(row, "O"))
|
||||
instrument = common.parse_security(
|
||||
_cell(row, "B"),
|
||||
asset_class_hint=self.asset_class_hint,
|
||||
venue=_text(row, "F") or None,
|
||||
currency=currency,
|
||||
extra={
|
||||
"nominal": _str_decimal(_cell(row, "R")),
|
||||
"coupon_rate": _str_decimal(_cell(row, "X")),
|
||||
"coupon_or_maturity_date": _iso(_cell(row, "U")),
|
||||
},
|
||||
)
|
||||
if instrument.isin is None:
|
||||
self.warnings.append(
|
||||
f"строка {line_no}: в остатках не разобран ISIN бумаги «{_text(row, 'B')[:60]}»"
|
||||
)
|
||||
self.instruments.append(instrument)
|
||||
self.positions.append(
|
||||
PositionEnd(
|
||||
instrument=instrument,
|
||||
qty=qty,
|
||||
price=to_decimal(_cell(row, "P")),
|
||||
market_value=to_decimal(_cell(row, "AF")) or to_decimal(_cell(row, "AB")),
|
||||
currency=currency,
|
||||
accrued_interest=to_decimal(_cell(row, "S")),
|
||||
)
|
||||
)
|
||||
return True
|
||||
|
||||
def _row_security_moves(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
"""Settlement of the deals already emitted — counted, never emitted (see module doc)."""
|
||||
day = to_date(_cell(row, "I"))
|
||||
if day is None:
|
||||
return False
|
||||
qty = to_decimal(_cell(row, "L"))
|
||||
if qty is None:
|
||||
self.warnings.append(f"строка {line_no}: движение бумаг без количества")
|
||||
return True
|
||||
kind = common.trade_side_kind(_cell(row, "O"))
|
||||
if kind is None:
|
||||
self.warnings.append(
|
||||
f"строка {line_no}: неизвестный тип операции с бумагами «{_text(row, 'O')}»"
|
||||
)
|
||||
signed = -abs(qty) if kind is EventKind.sell else abs(qty)
|
||||
key = common.parse_security(_cell(row, "B")).key()
|
||||
self.move_qty[key] = self.move_qty.get(key, Decimal(0)) + signed
|
||||
self.move_rows += 1
|
||||
return True
|
||||
|
||||
def _row_trades(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
if common.to_datetime(_cell(row, "C")) is None:
|
||||
return False
|
||||
self.section_seq += 1
|
||||
try:
|
||||
event = common.build_trade_event(
|
||||
security=_cell(row, "B"),
|
||||
side=_cell(row, "F"),
|
||||
trade_dt=_cell(row, "C"),
|
||||
settle_date=_cell(row, "T"),
|
||||
quantity=_cell(row, "H"),
|
||||
price=_cell(row, "J"),
|
||||
price_currency=_cell(row, "I"),
|
||||
settlement_currency=_cell(row, "L"),
|
||||
amount_gross=_cell(row, "M"),
|
||||
accrued_interest=_cell(row, "O"),
|
||||
fees=(_cell(row, "P"), _cell(row, "R")),
|
||||
trade_no=_cell(row, "Z"),
|
||||
venue=_cell(row, "AK"),
|
||||
order_no=_cell(row, "W"),
|
||||
exchange_trade_no=_cell(row, "AC"),
|
||||
counterparty=_cell(row, "AG"),
|
||||
raw_line_no=line_no,
|
||||
seq=self.section_seq,
|
||||
)
|
||||
except ValueError as exc:
|
||||
self.warnings.append(f"строка {line_no}: сделка пропущена — {exc}")
|
||||
return True
|
||||
|
||||
if not event.trade_no:
|
||||
self.warnings.append(
|
||||
f"строка {line_no}: сделка без «№ сделки», ключ дедупликации по отпечатку"
|
||||
)
|
||||
else:
|
||||
self.trade_numbers.append(event.trade_no)
|
||||
|
||||
gross = abs(to_decimal(_cell(row, "M")) or Decimal(0))
|
||||
self.trade_gross_signed += gross if event.kind is EventKind.sell else -gross
|
||||
self.trade_fee_total += event.fee or Decimal(0)
|
||||
if event.instrument is not None and event.quantity is not None:
|
||||
key = event.instrument.key()
|
||||
self.trade_qty[key] = self.trade_qty.get(key, Decimal(0)) + event.quantity
|
||||
self.instruments.append(event.instrument)
|
||||
self.events.append(event)
|
||||
return True
|
||||
|
||||
def _row_trades_completed(self, line_no: int, row: tuple[Any, ...]) -> bool:
|
||||
"""The same deals again (see module doc) — only their numbers are taken."""
|
||||
if common.to_datetime(_cell(row, "C")) is None:
|
||||
return False
|
||||
number = _text(row, "Z")
|
||||
if not number:
|
||||
self.warnings.append(
|
||||
f"строка {line_no}: завершённая сделка без «№ сделки» — сверка с "
|
||||
"«Заключенными» по ней невозможна"
|
||||
)
|
||||
return True
|
||||
self.completed_numbers.append(number)
|
||||
return True
|
||||
|
||||
# -- cross-checks --------------------------------------------------------------------
|
||||
|
||||
def _reconcile(self) -> None:
|
||||
"""Everything the report states twice, checked against itself.
|
||||
|
||||
A report is evidence: the value of parsing it is only as good as the assurance that
|
||||
what was extracted adds up to the totals the broker printed. Each mismatch below is a
|
||||
real bug class — a missed section, a sign flipped, a fee counted twice — so none of
|
||||
them may pass silently.
|
||||
"""
|
||||
concluded = set(self.trade_numbers)
|
||||
completed = set(self.completed_numbers)
|
||||
if len(concluded) != len(self.trade_numbers):
|
||||
self.warnings.append("в «Заключенных сделках» повторяются номера сделок")
|
||||
if completed and completed != concluded:
|
||||
only_completed = sorted(completed - concluded)
|
||||
only_concluded = sorted(concluded - completed)
|
||||
self.warnings.append(
|
||||
"номера сделок «Завершенных» и «Заключенных» расходятся: "
|
||||
f"только в завершённых {only_completed}, только в заключённых {only_concluded}"
|
||||
)
|
||||
|
||||
if self.move_rows and self.move_rows != len(self.trade_numbers):
|
||||
self.warnings.append(
|
||||
f"строк «Движения ценных бумаг» {self.move_rows}, сделок "
|
||||
f"{len(self.trade_numbers)} — расчёты не совпадают со сделками"
|
||||
)
|
||||
for key, qty in self.move_qty.items():
|
||||
traded = self.trade_qty.get(key, Decimal(0))
|
||||
if qty != traded:
|
||||
self.warnings.append(
|
||||
f"{key}: движение бумаг {qty} шт., сделки {traded} шт. — расхождение"
|
||||
)
|
||||
|
||||
settlement = self.summary.get(_SUMMARY_SETTLEMENT)
|
||||
if settlement is not None and abs(settlement - self.trade_gross_signed) > _MONEY_TOLERANCE:
|
||||
self.warnings.append(
|
||||
f"сальдо расчётов по сделкам в сводке {settlement}, по таблице сделок "
|
||||
f"{self.trade_gross_signed}"
|
||||
)
|
||||
if (
|
||||
self.cash_settlement_total
|
||||
and settlement is not None
|
||||
and abs(settlement - self.cash_settlement_total) > _MONEY_TOLERANCE
|
||||
):
|
||||
self.warnings.append(
|
||||
f"сальдо расчётов в сводке {settlement}, в движении денег "
|
||||
f"{self.cash_settlement_total}"
|
||||
)
|
||||
|
||||
self._reconcile_fees()
|
||||
self._reconcile_cash()
|
||||
|
||||
def _reconcile_fees(self) -> None:
|
||||
"""Deal commissions against «Вознаграждение брокера» in the cash section.
|
||||
|
||||
They agree to the kopeck in every report seen, which is what justifies capitalising
|
||||
the fee into the deal and dropping the cash line (`common.build_trade_event`). The
|
||||
check is what keeps that justification honest: if a fee ever belongs to no deal, the
|
||||
difference is emitted as its own `commission` event so the money does not vanish, and
|
||||
the warning says how much was moved.
|
||||
"""
|
||||
charged = -self.cash_fee_total # the cash column prints a charge as negative
|
||||
residual = common.broker_fee_residual(charged, self.trade_fee_total)
|
||||
summary_fee = self.summary.get(_SUMMARY_BROKER_FEE)
|
||||
if summary_fee is not None and abs(-summary_fee - charged) > _MONEY_TOLERANCE:
|
||||
self.warnings.append(
|
||||
f"вознаграждение брокера в сводке {summary_fee}, в движении денег "
|
||||
f"{self.cash_fee_total}"
|
||||
)
|
||||
if not residual:
|
||||
return
|
||||
|
||||
self.warnings.append(
|
||||
f"комиссия в движении денег {charged} не сходится с комиссиями по сделкам "
|
||||
f"{self.trade_fee_total}; разница {residual} записана отдельным событием commission"
|
||||
)
|
||||
row = self.last_fee_row
|
||||
day = to_date(_cell(row, "B")) if row is not None else None
|
||||
currency = (normalize_currency(_cell(row, "G")) if row is not None else None) or (
|
||||
self.cash_end[0].currency if self.cash_end else "RUB"
|
||||
)
|
||||
if day is None:
|
||||
self.warnings.append("разницу по комиссии не к чему привязать по дате — пропущена")
|
||||
return
|
||||
self.events.append(
|
||||
BrokerEvent(
|
||||
kind=EventKind.commission,
|
||||
trade_date=day,
|
||||
amount=-residual,
|
||||
currency=currency,
|
||||
fee=residual,
|
||||
fee_currency=currency,
|
||||
description="Вознаграждение брокера сверх комиссий по сделкам",
|
||||
meta={"section": "cash", "operation": "Вознаграждение брокера", "residual": True},
|
||||
)
|
||||
)
|
||||
|
||||
def _reconcile_cash(self) -> None:
|
||||
"""Opening balance plus every emitted event must equal the closing balance.
|
||||
|
||||
This is the one check that covers the whole file at once: it fails if a deposit was
|
||||
missed, if a trade's sign is wrong, or if a fee got counted twice. Note that it only
|
||||
holds because the deal commissions are inside `amount` — the same reason the cash fee
|
||||
lines are not events.
|
||||
"""
|
||||
opening = self.summary.get(_SUMMARY_OPENING)
|
||||
closing = self.summary.get(_SUMMARY_CLOSING)
|
||||
if opening is None or closing is None:
|
||||
return
|
||||
derived = opening + sum((event.amount for event in self.events), Decimal(0))
|
||||
if abs(derived - closing) > _MONEY_TOLERANCE:
|
||||
self.warnings.append(
|
||||
f"исходящий остаток по сводке {closing}, по разобранным событиям {derived}"
|
||||
)
|
||||
for balance in self.cash_end:
|
||||
if balance.currency == (self.summary_currency or "RUB") and (
|
||||
abs(balance.balance - closing) > _MONEY_TOLERANCE
|
||||
):
|
||||
self.warnings.append(
|
||||
f"остаток {balance.currency} {balance.balance} не совпадает со сводкой "
|
||||
f"{closing}"
|
||||
)
|
||||
|
||||
|
||||
def _str_decimal(value: Any) -> str | None:
|
||||
number = to_decimal(value)
|
||||
return None if number is None else str(number)
|
||||
|
||||
|
||||
def _iso(value: Any) -> str | None:
|
||||
day = to_date(value)
|
||||
return None if day is None else day.isoformat()
|
||||
|
||||
|
||||
parser = VtbXlsxParser()
|
||||
Reference in New Issue
Block a user