ada/fin-tracker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(ledger): импорт отчётов в леджер — приём, дедупликация, pending_instrument

Browse Source 
Поток: upload -> raw_report_file (sha256 UNIQUE) -> parse -> raw_report_line,
событий в леджере ещё нет -> preview -> POST /imports/{id}/commit ->
ledger/ingest.py резолвит инструмент, считает dedupe_key, пишет event
confirmed | shadow (plan §1.6 B) — сравнивая account.primary_event_source
с источником отчёта, а не гадая. Нерезолвленный инструмент ждёт в
pending_instrument, никогда не угадывается; POST /instruments/pending/{id}/resolve
привязывает и пересобирает лоты.

ledger/dedupe.py — shadow-матчинг случая B двумя проходами (точная дата, затем
±1 рабочий день, жадно 1:1, |price| ±0,5 %). Шаги shadow_dedupe и
report_reconcile зарегистрированы перед quality: оба говорят через FINDINGS.

/instruments/pending регистрируется в app.py ДО routers/instruments.py:
FastAPI сопоставляет маршруты по порядку, и /instruments/{id} с типом int
отвечает 422 на нечисловой сегмент, а не проваливается дальше.

Контракт — docs/ai/import-contract.md, общий для бэкенда и Flutter.
This commit is contained in:
Dmitry
2026-09-19 10:40:04 +03:00
parent 2e742a093b
commit ff3b76871d
16 changed files with 4484 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/fintracker/api/app.py
+6
View File
@@ -22,6 +22,7 @@ from fintracker.api.routers import (
categories,
events,
health,
imports,
instruments,
links,
metrics,
@@ -86,6 +87,11 @@ def create_app() -> FastAPI:
app.include_router(cashflow.router, prefix=API_PREFIX)
app.include_router(analytics.router, prefix=API_PREFIX)
app.include_router(events.router, prefix=API_PREFIX)
app.include_router(imports.router, prefix=API_PREFIX)
# Before `instruments`: FastAPI matches in registration order, and `/instruments/{id}`
# typed `int` does not fall through on a non-numeric segment — it answers 422. So
# `/instruments/pending` has to be declared first or it becomes unreachable.
app.include_router(imports.pending_router, prefix=API_PREFIX)
app.include_router(instruments.router, prefix=API_PREFIX)
app.include_router(links.router, prefix=API_PREFIX)
app.include_router(metrics.router, prefix=API_PREFIX)
backend/src/fintracker/api/routers/imports.py
+380
View File
@@ -0,0 +1,380 @@
"""Broker report imports and the resolution of what they named (plan §4).
Implements `docs/ai/import-contract.md` literally — paths, field names and error codes are
what the Flutter screens were written against, so this module's job is translation, not
judgement: everything that decides anything lives in `ledger/report_import.py`.
Two routers, one module, on purpose. The pending-instrument endpoints belong to the import
feature but sit under `/instruments` because that is where the user looks for an instrument,
and `api/routers/instruments.py` has nothing to do with importing.
**Mounting order matters.** `pending_router` must be included BEFORE `instruments.router`:
FastAPI matches in registration order, and `/instruments/{instrument_id}` would otherwise
swallow `/instruments/pending` and fail to read "pending" as an int.
"""
from __future__ import annotations
from typing import Annotated
from fastapi import APIRouter, File, Form, Query, UploadFile
from sqlalchemy import select
from fintracker.api.deps import CurrentUser, SessionDep
from fintracker.api.errors import Problem
from fintracker.api.schemas.imports import (
AccountSuggestion,
CommitRequest,
ImportCounts,
ImportPreview,
ImportResult,
ImportSummary,
PendingInstrumentOut,
PendingResolveRequest,
PendingResolveResult,
ReconciliationCash,
ReconciliationOut,
ReconciliationPosition,
SampleEventOut,
)
from fintracker.ledger import report_import
from fintracker.ledger.report_import import (
CommitOutcome,
ImportProblem,
InstrumentSpec,
Preview,
Reconciliation,
)
from fintracker.models import Account, PendingInstrument, PendingInstrumentStatus, RawReportFile
router = APIRouter(prefix="/imports", tags=["imports"])
pending_router = APIRouter(prefix="/instruments", tags=["instruments"])
#: `pending_router` must be included BEFORE `routers/instruments.py` in `api/app.py`:
#: FastAPI matches in registration order, and `/instruments/{instrument_id}` typed `int`
#: answers 422 on a non-numeric segment instead of falling through to the next route.
#: `tests/api/test_imports_api.py` guards the ordering.
def _problem(exc: ImportProblem) -> Problem:
return Problem(exc.status_code, exc.title, exc.detail, extra=exc.extra)
@router.post("", name="create")
async def create_import(
session: SessionDep,
_: CurrentUser,
file: Annotated[UploadFile, File(description="the report itself")],
account_id: Annotated[int | None, Form(description="target account, if known")] = None,
parser: Annotated[str | None, Form(description="force a parser from registry.names()")] = None,
) -> ImportPreview:
"""Upload a report, parse it and show what committing it would do. Writes no event."""
data = await file.read()
try:
outcome = await report_import.upload(
session,
data=data,
filename=file.filename or "report",
account_id=account_id,
parser_name=parser,
)
preview = await report_import.build_preview(
session, outcome.file, duplicate_of_id=outcome.duplicate_of_id
)
except ImportProblem as exc:
raise _problem(exc) from exc
return _preview_out(preview)
@router.get("", name="list")
async def list_imports(
session: SessionDep,
_: CurrentUser,
status: Annotated[
str | None, Query(description="uploaded | parsed | committed | failed")
] = None,
limit: Annotated[int, Query(ge=1, le=200)] = 50,
offset: Annotated[int, Query(ge=0)] = 0,
) -> list[ImportSummary]:
"""Newest imports first, with the counters each was committed with."""
stmt = select(RawReportFile).order_by(RawReportFile.id.desc()).limit(limit).offset(offset)
if status is not None:
stmt = stmt.where(RawReportFile.parse_status == status)
rows = (await session.execute(stmt)).scalars().all()
names = await _account_names(session, [r.account_id for r in rows])
return [
_summary_out(r, names.get(r.account_id) if r.account_id is not None else None) for r in rows
]
@router.get("/{import_id}", name="get")
async def get_import(session: SessionDep, _: CurrentUser, import_id: int) -> ImportPreview:
"""The preview again — recomputed for an uncommitted import, replayed for a committed one."""
row = await _load(session, import_id)
try:
preview = await report_import.build_preview(session, row)
except ImportProblem as exc:
raise _problem(exc) from exc
return _preview_out(preview)
@router.post("/{import_id}/commit", name="commit")
async def commit_import(
session: SessionDep, _: CurrentUser, import_id: int, body: CommitRequest
) -> ImportResult:
"""Write the file's events into the ledger and rebuild every metric."""
row = await _load(session, import_id)
try:
outcome = await report_import.commit(
session,
row,
account_id=body.account_id,
confirm_duplicates=body.confirm_duplicates,
dry_run=body.dry_run,
)
except ImportProblem as exc:
raise _problem(exc) from exc
return _result_out(import_id, outcome)
@router.delete("/{import_id}", name="delete", status_code=204)
async def delete_import(session: SessionDep, _: CurrentUser, import_id: int) -> None:
"""Drop an uncommitted import. A committed one stays: `raw_*` is append-only."""
row = await _load(session, import_id)
if row.parse_status == "committed":
raise Problem(
409,
"Conflict",
f"Импорт #{import_id} закоммичен — его события уже в леджере, след не удаляется",
)
await session.delete(row)
await session.commit()
@pending_router.get("/pending", name="pending")
async def list_pending(
session: SessionDep,
_: CurrentUser,
status: Annotated[str, Query(description="pending | resolved | ignored | all")] = "pending",
limit: Annotated[int, Query(ge=1, le=500)] = 100,
offset: Annotated[int, Query(ge=0)] = 0,
) -> list[PendingInstrumentOut]:
"""Instruments a report named that nothing in the master resolves to."""
stmt = (
select(PendingInstrument)
.order_by(PendingInstrument.occurrences.desc(), PendingInstrument.id)
.limit(limit)
.offset(offset)
)
if status != "all":
if status not in {s.value for s in PendingInstrumentStatus}:
raise Problem(
400, "Bad Request", f"Неизвестный статус {status!r}: pending | resolved | ignored"
)
stmt = stmt.where(PendingInstrument.status == status)
rows = (await session.execute(stmt)).scalars().all()
return [_pending_out(r) for r in rows]
@pending_router.post("/pending/{pending_id}/resolve", name="pending_resolve")
async def resolve_pending(
session: SessionDep, _: CurrentUser, pending_id: int, body: PendingResolveRequest
) -> PendingResolveResult:
"""Link, create or ignore — then bind the events that were waiting and rebuild the lots."""
row = await session.get(PendingInstrument, pending_id)
if row is None:
raise Problem(404, "Not Found", f"Нет строки pending_instrument #{pending_id}")
spec = None
if body.instrument is not None:
spec = InstrumentSpec(
asset_class=body.instrument.asset_class,
name=body.instrument.name,
currency=body.instrument.currency,
isin=body.instrument.isin,
ticker=body.instrument.ticker,
board=body.instrument.board,
lot=body.instrument.lot,
)
try:
outcome = await report_import.resolve_pending(
session, row, action=body.action, instrument_id=body.instrument_id, instrument=spec
)
except ImportProblem as exc:
raise _problem(exc) from exc
return PendingResolveResult(
id=outcome.pending.id,
status=outcome.pending.status.value,
instrument_id=outcome.instrument_id,
events_bound=outcome.events_bound,
alias_created=outcome.alias_created,
metrics_refreshed=outcome.metrics_refreshed,
)
async def _load(session: SessionDep, import_id: int) -> RawReportFile:
row = await session.get(RawReportFile, import_id)
if row is None:
raise Problem(404, "Not Found", f"Нет импорта #{import_id}")
return row
async def _account_names(session: SessionDep, ids: list[int | None]) -> dict[int, str]:
wanted = {i for i in ids if i is not None}
if not wanted:
return {}
rows = (
await session.execute(select(Account.id, Account.name).where(Account.id.in_(wanted)))
).all()
return {account_id: name for account_id, name in rows}
def _counts_out(counts: dict) -> ImportCounts:
return ImportCounts.model_validate({**_ZERO_COUNTS, **(counts or {})})
_ZERO_COUNTS = {
"lines": 0,
"events_total": 0,
"events_new": 0,
"events_duplicate": 0,
"events_shadow": 0,
"events_pending": 0,
"by_kind": {},
}
def _summary_out(
row: RawReportFile, account_name: str | None, duplicate_of_id: int | None = None
) -> ImportSummary:
return ImportSummary(
id=row.id,
broker=row.broker,
filename=row.filename,
sha256=row.sha256,
size_bytes=row.size_bytes,
parser_name=row.parser_name,
parser_version=row.parser_version,
parse_status=row.parse_status.value,
error=row.error,
duplicate_of_id=duplicate_of_id,
account_external_id=row.account_external_id,
account_id=row.account_id,
account_name=account_name,
period_from=row.period_from,
period_to=row.period_to,
uploaded_at=row.uploaded_at,
committed_at=row.committed_at,
counts=_counts_out(row.counts or {}),
)
def _preview_out(preview: Preview) -> ImportPreview:
row = preview.file
base = _summary_out(
row,
preview.account.name if preview.account else None,
duplicate_of_id=preview.duplicate_of_id,
)
return ImportPreview(
**base.model_dump(exclude={"counts"}),
counts=_counts_out(preview.counts),
account_suggestions=[
AccountSuggestion(
id=a.id,
name=a.name,
broker=a.broker.value if a.broker else None,
source_id=a.source_id,
)
for a in preview.suggestions
],
pending_instruments=[_pending_out(p) for p in preview.pending],
reconciliation=_recon_out(preview.reconciliation),
warnings=preview.warnings,
sample_events=[
SampleEventOut(
line_no=s.line_no,
kind=s.kind,
trade_date=s.trade_date,
settle_date=s.settle_date,
instrument_key=s.instrument_key,
instrument_name=s.instrument_name,
instrument_id=s.instrument_id,
quantity=s.quantity,
price=s.price,
amount=s.amount,
currency=s.currency,
fee=s.fee,
trade_no=s.trade_no,
dedupe_key=s.dedupe_key,
is_duplicate=s.is_duplicate,
description=s.description,
)
for s in preview.sample_events
],
)
def _recon_out(recon: Reconciliation) -> ReconciliationOut:
return ReconciliationOut(
as_of=recon.as_of,
positions=[
ReconciliationPosition(
instrument_id=p.instrument_id,
instrument_name=p.instrument_name,
ticker=p.ticker,
isin=p.isin,
qty_report=p.qty_report,
qty_derived=p.qty_derived,
qty_delta=p.qty_delta,
matches=p.matches,
)
for p in recon.positions
],
cash=[
ReconciliationCash(
currency=c.currency,
balance_report=c.balance_report,
balance_derived=c.balance_derived,
delta=c.delta,
matches=c.matches,
)
for c in recon.cash
],
matches=recon.matches,
)
def _pending_out(row: PendingInstrument) -> PendingInstrumentOut:
return PendingInstrumentOut(
id=row.id,
source=row.source,
source_key=row.source_key,
isin=row.isin,
ticker=row.ticker,
board=row.board,
name=row.name,
currency=row.currency,
asset_class_hint=row.asset_class_hint,
occurrences=row.occurrences,
sample_quantity=row.sample_quantity,
sample_price=row.sample_price,
status=row.status.value,
instrument_id=row.instrument_id,
first_seen_file_id=row.first_seen_file_id,
created_at=row.created_at,
)
def _result_out(import_id: int, outcome: CommitOutcome) -> ImportResult:
return ImportResult(
import_id=import_id,
committed=outcome.committed,
events_created=outcome.events_created,
events_updated=outcome.events_updated,
events_skipped=outcome.events_skipped,
events_shadow=outcome.events_shadow,
pending_instruments=outcome.pending_instruments,
reconciliation=_recon_out(outcome.reconciliation),
metrics_refreshed=outcome.metrics_refreshed,
)
backend/src/fintracker/api/schemas/imports.py
+200
View File
@@ -0,0 +1,200 @@
"""Schemas for `/imports` and `/instruments/pending`, per `docs/ai/import-contract.md`.
Two conventions the Flutter client depends on, both repeated from AGENTS.md because breaking
either one breaks the generated Dart rather than a test:
* money and quantities are `Money` / `MoneyOpt` — `Decimal` in Python, a string on the wire;
* every stable key (`broker`, `parse_status`, `kind`, `status`, `asset_class`) is a plain
`str`, never an enum. `AssetClass.index` cannot exist as a Dart enum member (it collides
with `Enum.index`) and one enum leaking out stops the whole client from compiling.
"""
from __future__ import annotations
from datetime import date, datetime
from typing import Literal
from pydantic import BaseModel, Field
from fintracker.api.schemas.common import Money, MoneyOpt
class AccountSuggestion(BaseModel):
"""A broker account the user may attach an import to when the report's number matched none."""
id: int
name: str
broker: str | None
"""tinvest | sber | vtb | other"""
source_id: str
class ImportCounts(BaseModel):
lines: int
events_total: int
events_new: int
"""No `event` carries this row's `dedupe_key` yet."""
events_duplicate: int
"""The key is already in the ledger: committing upserts, it does not insert."""
events_shadow: int
"""Rows that will land as `shadow` — this feed is not the account's primary source."""
events_pending: int
"""Rows whose instrument nobody recognises yet."""
by_kind: dict[str, int]
class PendingInstrumentOut(BaseModel):
id: int
source: str
source_key: str
isin: str | None = None
ticker: str | None = None
board: str | None = None
name: str | None = None
currency: str | None = None
asset_class_hint: str | None = None
"""share | bond | etf | fund | currency — the report's own wording, never applied alone."""
occurrences: int
sample_quantity: MoneyOpt = None
sample_price: MoneyOpt = None
status: str
"""pending | resolved | ignored"""
instrument_id: int | None = None
first_seen_file_id: int | None = None
created_at: datetime | None = None
class ReconciliationPosition(BaseModel):
instrument_id: int | None
instrument_name: str
ticker: str | None
isin: str | None
qty_report: Money
qty_derived: MoneyOpt
"""Null when the account is unknown or its lots were never rebuilt — never a zero."""
qty_delta: MoneyOpt
matches: bool
class ReconciliationCash(BaseModel):
currency: str
balance_report: Money
balance_derived: MoneyOpt
delta: MoneyOpt
matches: bool
class ReconciliationOut(BaseModel):
as_of: date | None
positions: list[ReconciliationPosition] = Field(default_factory=list)
cash: list[ReconciliationCash] = Field(default_factory=list)
matches: bool = True
class SampleEventOut(BaseModel):
line_no: int
kind: str
"""buy | sell | dividend | coupon | … — the `EventKind` value as a string."""
trade_date: date
settle_date: date | None = None
instrument_key: str | None = None
instrument_name: str | None = None
instrument_id: int | None = None
quantity: MoneyOpt = None
price: MoneyOpt = None
amount: Money
currency: str
fee: MoneyOpt = None
trade_no: str | None = None
dedupe_key: str
is_duplicate: bool
description: str | None = None
class ImportSummary(BaseModel):
"""One import without the heavy parts — what `GET /imports` lists."""
id: int
broker: str
"""sber | vtb | csv"""
filename: str
sha256: str
size_bytes: int
parser_name: str | None = None
parser_version: str | None = None
parse_status: str
"""uploaded | parsed | committed | failed"""
error: str | None = None
duplicate_of_id: int | None = None
"""Not null ⇒ these bytes were already uploaded, and this is that import."""
account_external_id: str | None = None
account_id: int | None = None
"""Null ⇒ the account is unknown and commit is refused."""
account_name: str | None = None
period_from: date | None = None
period_to: date | None = None
uploaded_at: datetime | None = None
committed_at: datetime | None = None
counts: ImportCounts
class ImportPreview(ImportSummary):
"""The full preview: `POST /imports` and `GET /imports/{id}` both return this."""
account_suggestions: list[AccountSuggestion] = Field(default_factory=list)
pending_instruments: list[PendingInstrumentOut] = Field(default_factory=list)
reconciliation: ReconciliationOut
warnings: list[str] = Field(default_factory=list)
sample_events: list[SampleEventOut] = Field(default_factory=list)
class CommitRequest(BaseModel):
account_id: int | None = None
"""Needed only when the preview could not work the account out on its own."""
confirm_duplicates: bool = False
dry_run: bool = False
class ImportResult(BaseModel):
import_id: int
committed: bool
events_created: int
events_updated: int
events_skipped: int
"""Rows still waiting on a `pending_instrument`."""
events_shadow: int
pending_instruments: int
reconciliation: ReconciliationOut
metrics_refreshed: bool
class NewInstrument(BaseModel):
"""The instrument to create from a report's own data (`action = "create"`)."""
asset_class: str
"""share | bond | etf | fund | currency | index | deposit | real_estate | crypto | custom"""
name: str
currency: str = "RUB"
isin: str | None = None
ticker: str | None = None
board: str | None = None
lot: int = 1
class PendingResolveRequest(BaseModel):
action: Literal["link", "create", "ignore"]
instrument_id: int | None = None
"""Required for `link`."""
instrument: NewInstrument | None = None
"""Required for `create`."""
class PendingResolveResult(BaseModel):
id: int
status: str
"""resolved | ignored"""
instrument_id: int | None = None
events_bound: int
"""How many events left `pending` for `confirmed`/`shadow`."""
alias_created: bool
metrics_refreshed: bool