feat(sources): контракт источников, worker и синк ZenMoney + ЦБ
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
This commit is contained in:
Dmitry
@@ -0,0 +1,13 @@
|
||||
"""Data sources.
|
||||
|
||||
Importing this package pulls in every source package, and each of those calls
|
||||
`registry.register(...)` at import time — so the API, the worker and the CLI all see the
|
||||
same set of sources just by importing `fintracker.sources`.
|
||||
|
||||
`sources/base.py` and `sources/registry.py` must never import the source packages back:
|
||||
that is what keeps this import graph acyclic.
|
||||
"""
|
||||
|
||||
from fintracker.sources import cbr, moex, tinvest, zenmoney
|
||||
|
||||
__all__ = ["cbr", "moex", "tinvest", "zenmoney"]
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Contract every data source implements (plan §2.3).
|
||||
|
||||
A source pulls from one upstream (ZenMoney, T-Invest, MOEX, CBR) into the raw tier and
|
||||
optionally maps into core tables. It owns its cursor via `SyncContext` and reports what
|
||||
it did in `SyncResult`; the worker handles locking, run logging and scheduling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.config import Settings
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
cursor_after: str | None = None
|
||||
counts: dict[str, int] = field(default_factory=dict)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
changed: bool = True
|
||||
"""False when nothing new landed — lets the worker skip the metrics refresh."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncContext:
|
||||
session: AsyncSession
|
||||
settings: Settings
|
||||
cursor_before: str | None
|
||||
triggered_by: str
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class Source(Protocol):
|
||||
name: str
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult: ...
|
||||
@@ -0,0 +1,8 @@
|
||||
"""CBR FX source package. Importing it registers the source."""
|
||||
|
||||
from fintracker.sources.cbr.sync import CbrSource
|
||||
from fintracker.sources.registry import register
|
||||
|
||||
register(CbrSource())
|
||||
|
||||
__all__ = ["CbrSource"]
|
||||
@@ -0,0 +1,163 @@
|
||||
"""CBR (Bank of Russia) official FX rates over the public XML scripts.
|
||||
|
||||
Two endpoints:
|
||||
|
||||
* `GET /scripts/XML_daily.asp?date_req=DD/MM/YYYY` — every quoted currency on one day;
|
||||
used only for the `CharCode -> internal id` map ('USD' -> 'R01235').
|
||||
* `GET /scripts/XML_dynamic.asp?date_req1=…&date_req2=…&VAL_NM_RQ=R01235` — one currency
|
||||
over a date range, `<Record Date="01.09.2026"><Nominal>1</Nominal><Value>92,1234</Value>`.
|
||||
|
||||
Both answer windows-1251 XML with a decimal comma, and quote RUB per `Nominal` units
|
||||
(100 for JPY, 10 for CNY at times). Values are parsed into `Decimal` and kept exactly as
|
||||
printed, nominal included — dividing per unit happens later in `pricing/fx.py`.
|
||||
|
||||
NETWORK NOTE: `trust_env=False`. This host exports http_proxy/https_proxy/all_proxy, and
|
||||
cbr.ru fails the TLS handshake through that proxy, so the CBR client must bypass the
|
||||
environment entirely. (ZenMoney is the opposite case and keeps the proxy — see
|
||||
`sources/zenmoney/client.py`.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
import httpx
|
||||
|
||||
DAILY_URL = "https://www.cbr.ru/scripts/XML_daily.asp"
|
||||
DYNAMIC_URL = "https://www.cbr.ru/scripts/XML_dynamic.asp"
|
||||
ENCODING = "windows-1251"
|
||||
TIMEOUT = 60.0
|
||||
MAX_RANGE_DAYS = 366
|
||||
"""CBR copes with longer ranges, but requests are chunked to at most a year to be polite."""
|
||||
|
||||
|
||||
class CbrError(RuntimeError):
|
||||
"""CBR refused or returned something unparsable."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CbrQuote:
|
||||
rate_date: date
|
||||
nominal: int
|
||||
value: Decimal
|
||||
"""RUB per `nominal` units, exactly as CBR printed it."""
|
||||
|
||||
|
||||
def new_http_client() -> httpx.AsyncClient:
|
||||
# trust_env=False: see NETWORK NOTE above
|
||||
return httpx.AsyncClient(trust_env=False, timeout=TIMEOUT)
|
||||
|
||||
|
||||
def _fmt(day: date) -> str:
|
||||
return day.strftime("%d/%m/%Y")
|
||||
|
||||
|
||||
def _decimal(text: str | None) -> Decimal | None:
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return Decimal(text.replace("\xa0", "").replace(" ", "").replace(",", "."))
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_xml(payload: bytes) -> ET.Element:
|
||||
try:
|
||||
return ET.fromstring(payload.decode(ENCODING))
|
||||
except (ET.ParseError, UnicodeDecodeError) as exc:
|
||||
raise CbrError(f"CBR returned unparsable XML: {exc}") from exc
|
||||
|
||||
|
||||
def parse_code_map(payload: bytes) -> dict[str, str]:
|
||||
"""XML_daily -> {'USD': 'R01235', …}."""
|
||||
root = _parse_xml(payload)
|
||||
codes: dict[str, str] = {}
|
||||
for node in root.iter("Valute"):
|
||||
char_code = (node.findtext("CharCode") or "").strip().upper()
|
||||
cbr_id = (node.get("ID") or "").strip()
|
||||
if char_code and cbr_id:
|
||||
codes[char_code] = cbr_id
|
||||
return codes
|
||||
|
||||
|
||||
def parse_dynamic(payload: bytes) -> list[CbrQuote]:
|
||||
"""XML_dynamic -> quotes, nominal and value untouched."""
|
||||
root = _parse_xml(payload)
|
||||
quotes: list[CbrQuote] = []
|
||||
for node in root.iter("Record"):
|
||||
raw_date = node.get("Date")
|
||||
value = _decimal(node.findtext("Value"))
|
||||
if not raw_date or value is None:
|
||||
continue
|
||||
try:
|
||||
rate_date = datetime.strptime(raw_date, "%d.%m.%Y").date()
|
||||
except ValueError:
|
||||
continue
|
||||
nominal_text = (node.findtext("Nominal") or "1").strip()
|
||||
try:
|
||||
nominal = int(nominal_text.replace("\xa0", "").replace(" ", "")) or 1
|
||||
except ValueError:
|
||||
nominal = 1
|
||||
quotes.append(CbrQuote(rate_date=rate_date, nominal=nominal, value=value))
|
||||
return quotes
|
||||
|
||||
|
||||
def split_range(start: date, end: date, max_days: int = MAX_RANGE_DAYS) -> list[tuple[date, date]]:
|
||||
"""Chunk a long range into windows of at most `max_days` days."""
|
||||
if start > end:
|
||||
return []
|
||||
windows: list[tuple[date, date]] = []
|
||||
window_start = start
|
||||
while window_start <= end:
|
||||
window_end = min(end, window_start + timedelta(days=max_days - 1))
|
||||
windows.append((window_start, window_end))
|
||||
window_start = window_end + timedelta(days=1)
|
||||
return windows
|
||||
|
||||
|
||||
class CbrClient:
|
||||
def __init__(self, http: httpx.AsyncClient | None = None) -> None:
|
||||
self._http = http
|
||||
self._owns_http = http is None
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
if self._http is None:
|
||||
self._http = new_http_client()
|
||||
return self._http
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._http is not None and self._owns_http:
|
||||
await self._http.aclose()
|
||||
self._http = None
|
||||
|
||||
async def __aenter__(self) -> CbrClient:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info: object) -> None:
|
||||
await self.aclose()
|
||||
|
||||
async def _get(self, url: str, params: dict[str, str]) -> bytes:
|
||||
resp = await self._client().get(url, params=params)
|
||||
if resp.status_code >= 400:
|
||||
raise CbrError(f"{url} failed: HTTP {resp.status_code}")
|
||||
return resp.content
|
||||
|
||||
async def code_map(self, on: date) -> dict[str, str]:
|
||||
return parse_code_map(await self._get(DAILY_URL, {"date_req": _fmt(on)}))
|
||||
|
||||
async def quotes(self, cbr_id: str, start: date, end: date) -> list[CbrQuote]:
|
||||
collected: list[CbrQuote] = []
|
||||
for window_start, window_end in split_range(start, end):
|
||||
payload = await self._get(
|
||||
DYNAMIC_URL,
|
||||
{
|
||||
"date_req1": _fmt(window_start),
|
||||
"date_req2": _fmt(window_end),
|
||||
"VAL_NM_RQ": cbr_id,
|
||||
},
|
||||
)
|
||||
collected.extend(parse_dynamic(payload))
|
||||
return collected
|
||||
@@ -0,0 +1,161 @@
|
||||
"""The `cbr` source: official RUB rates for every currency the ledger actually uses.
|
||||
|
||||
What to fetch is derived from the data, not configured: the distinct currencies of
|
||||
`account.currency`, `cash_txn.income_currency` and `cash_txn.outcome_currency` (RUB itself
|
||||
is quoted as 1.0 by `pricing/fx.py`, so it is never requested).
|
||||
|
||||
Date range: from the earliest transaction minus 7 days (or today − 30 days when there are
|
||||
no transactions yet) up to today — `analytics.today_local()`, the deployment timezone, the
|
||||
same "today" the metrics end on. MSK runs ahead of UTC, so a UTC date would ask for one day
|
||||
less for the first hours of every Moscow morning. Incrementally, the cursor is the last date
|
||||
fetched and the next run starts 3 days earlier — CBR publishes the next business day's rate
|
||||
in the evening, and that overlap re-reads the tail cheaply. If the ledger grew *backwards* (an
|
||||
import of older history) the full range is used again, detected by comparing the wanted
|
||||
start with the earliest date already in `raw_cbr_rate`.
|
||||
|
||||
Currencies CBR does not quote (crypto, metals such as XAU) are reported in `warnings` and
|
||||
skipped — never an error, and never an invented rate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import ledger_date_range, today_local
|
||||
from fintracker.models import Account, CashTxn, RawCbrRate
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.cbr.client import CbrClient, CbrQuote
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = "cbr"
|
||||
BASE = "RUB"
|
||||
LOOKBACK_DAYS = 7
|
||||
"""Margin before the first transaction, so a rate exists for its date even over a weekend."""
|
||||
CURSOR_OVERLAP_DAYS = 3
|
||||
BACKFILL_TOLERANCE_DAYS = 14
|
||||
"""How far the first stored quote may legitimately sit after the wanted start (holidays)."""
|
||||
NO_HISTORY_DAYS = 30
|
||||
CHUNK = 500
|
||||
|
||||
|
||||
class CbrSource:
|
||||
name = SOURCE
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
session = ctx.session
|
||||
today = today_local()
|
||||
currencies = await currencies_in_use(session)
|
||||
if not currencies:
|
||||
log.info("cbr: only %s in use, nothing to fetch", BASE)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts={"currencies": 0, "rates": 0},
|
||||
changed=False,
|
||||
)
|
||||
|
||||
start = await _start_date(session, ctx.cursor_before, today)
|
||||
warnings: list[str] = []
|
||||
stored = 0
|
||||
async with CbrClient() as client:
|
||||
codes = await client.code_map(today)
|
||||
fetched_currencies = 0
|
||||
for ccy in currencies:
|
||||
cbr_id = codes.get(ccy)
|
||||
if not cbr_id:
|
||||
warnings.append(f"{ccy} is not quoted by CBR — skipped (crypto or metal?)")
|
||||
continue
|
||||
quotes = await client.quotes(cbr_id, start, today)
|
||||
stored += await _store(session, ccy, quotes)
|
||||
fetched_currencies += 1
|
||||
|
||||
await session.commit()
|
||||
log.info(
|
||||
"cbr: %s rates for %s currencies, %s..%s%s",
|
||||
stored,
|
||||
fetched_currencies,
|
||||
start,
|
||||
today,
|
||||
f", skipped {len(warnings)}" if warnings else "",
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts={"currencies": fetched_currencies, "rates": stored},
|
||||
warnings=warnings,
|
||||
changed=stored > 0,
|
||||
)
|
||||
|
||||
|
||||
async def currencies_in_use(session: AsyncSession) -> list[str]:
|
||||
found: set[str] = set()
|
||||
rows = await session.execute(select(Account.currency).where(Account.currency.is_not(None)))
|
||||
found.update(code for code in rows.scalars().all() if code)
|
||||
# deleted transactions are tombstones no metric reads — their currency is not "in use"
|
||||
for column in (CashTxn.income_currency, CashTxn.outcome_currency):
|
||||
rows = await session.execute(
|
||||
select(column).where(column.is_not(None), CashTxn.deleted.is_(False)).distinct()
|
||||
)
|
||||
found.update(code for code in rows.scalars().all() if code)
|
||||
return sorted(code for code in found if code != BASE)
|
||||
|
||||
|
||||
async def _start_date(session: AsyncSession, cursor: str | None, today: date) -> date:
|
||||
first_txn, _ = await ledger_date_range(session)
|
||||
full_start = (
|
||||
first_txn - timedelta(days=LOOKBACK_DAYS)
|
||||
if first_txn is not None
|
||||
else today - timedelta(days=NO_HISTORY_DAYS)
|
||||
)
|
||||
cursor_date = _parse_date(cursor)
|
||||
if cursor_date is None:
|
||||
return full_start
|
||||
have_from = (await session.execute(select(func.min(RawCbrRate.rate_date)))).scalar_one_or_none()
|
||||
if have_from is None or full_start + timedelta(days=BACKFILL_TOLERANCE_DAYS) < have_from:
|
||||
# the ledger grew backwards (older history imported) — refetch from the new beginning.
|
||||
# The tolerance absorbs the normal case where the wanted start simply falls on a
|
||||
# weekend or a holiday stretch that CBR never quoted.
|
||||
return full_start
|
||||
return min(today, cursor_date - timedelta(days=CURSOR_OVERLAP_DAYS))
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError:
|
||||
log.warning("cbr: unusable cursor %r, refetching the full range", value)
|
||||
return None
|
||||
|
||||
|
||||
async def _store(session: AsyncSession, ccy: str, quotes: list[CbrQuote]) -> int:
|
||||
rows = [
|
||||
{
|
||||
"rate_date": quote.rate_date,
|
||||
"ccy": ccy,
|
||||
"nominal": quote.nominal,
|
||||
"value": quote.value,
|
||||
"fetched_at": datetime.now(UTC),
|
||||
}
|
||||
for quote in quotes
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
for start in range(0, len(rows), CHUNK):
|
||||
chunk = rows[start : start + CHUNK]
|
||||
stmt = pg_insert(RawCbrRate).values(chunk)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["rate_date", "ccy"],
|
||||
set_={
|
||||
"nominal": stmt.excluded.nominal,
|
||||
"value": stmt.excluded.value,
|
||||
"fetched_at": stmt.excluded.fetched_at,
|
||||
},
|
||||
)
|
||||
await session.execute(stmt)
|
||||
return len(rows)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Registry of available sources. Later phases register zenmoney, cbr, tinvest, moex here."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fintracker.sources.base import Source
|
||||
|
||||
_SOURCES: dict[str, Source] = {}
|
||||
|
||||
|
||||
def register(source: Source) -> Source:
|
||||
_SOURCES[source.name] = source
|
||||
return source
|
||||
|
||||
|
||||
def unregister(name: str) -> None:
|
||||
_SOURCES.pop(name, None)
|
||||
|
||||
|
||||
def get(name: str) -> Source:
|
||||
return _SOURCES[name]
|
||||
|
||||
|
||||
def names() -> list[str]:
|
||||
return sorted(_SOURCES)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""ZenMoney source package. Importing it registers the source."""
|
||||
|
||||
from fintracker.sources.registry import register
|
||||
from fintracker.sources.zenmoney.sync import ZenmoneySource
|
||||
|
||||
register(ZenmoneySource())
|
||||
|
||||
__all__ = ["ZenmoneySource"]
|
||||
@@ -0,0 +1,280 @@
|
||||
"""HTTP access to the ZenMoney API (plan § "ZenMoney").
|
||||
|
||||
Reading is possible only through one endpoint: `POST https://api.zenmoney.ru/v8/diff/`.
|
||||
It is incremental by `serverTimestamp`; the first call (cursor 0) additionally asks for
|
||||
`forceFetch` of every entity type. The answer holds the same entity arrays (only rows
|
||||
changed since the cursor), a `deletion` list and the new `serverTimestamp`.
|
||||
|
||||
NETWORK NOTE: this host exports http_proxy/https_proxy/all_proxy and api.zenmoney.ru IS
|
||||
reachable through that proxy, so the client keeps httpx's default `trust_env=True`.
|
||||
The CBR client does the opposite — cbr.ru fails the TLS handshake through the proxy,
|
||||
see `sources/cbr/client.py`.
|
||||
|
||||
AUTH. Access tokens live 86400 s. Two modes, picked from settings:
|
||||
|
||||
* static token — `ZENMONEY_TOKEN` (e.g. minted at zerro.app/token). There is nothing to
|
||||
rotate, so HTTP 401 can only be reported as "renew ZENMONEY_TOKEN".
|
||||
* OAuth rotation — enabled when `ZENMONEY_CLIENT_ID` and `ZENMONEY_CLIENT_SECRET` are set.
|
||||
The `{access_token, refresh_token, expires_at}` triple lives in `source_credential`
|
||||
(source='zenmoney'), seeded from `ZENMONEY_REFRESH_TOKEN`. Before a sync an expired
|
||||
(or missing) access token is refreshed with
|
||||
`POST https://api.zenmoney.ru/oauth2/token/` form-encoded
|
||||
`grant_type=refresh_token&refresh_token=…&client_id=…&client_secret=…`; the response
|
||||
`{access_token, refresh_token, expires_in, token_type}` replaces the stored pair.
|
||||
A 401 on the diff call triggers exactly one extra refresh + retry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import TracebackType
|
||||
from typing import Any, Self
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.config import Settings
|
||||
from fintracker.models import SourceCredential
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = "zenmoney"
|
||||
DIFF_URL = "https://api.zenmoney.ru/v8/diff/"
|
||||
TOKEN_URL = "https://api.zenmoney.ru/oauth2/token/"
|
||||
TIMEOUT = 120.0
|
||||
|
||||
ENTITY_TYPES: tuple[str, ...] = (
|
||||
"instrument",
|
||||
"company",
|
||||
"user",
|
||||
"account",
|
||||
"tag",
|
||||
"merchant",
|
||||
"budget",
|
||||
"reminder",
|
||||
"reminderMarker",
|
||||
"transaction",
|
||||
)
|
||||
"""Every entity type /v8/diff/ can return; also the `forceFetch` list on a full pull."""
|
||||
|
||||
EXPIRY_SKEW = timedelta(minutes=5)
|
||||
DEFAULT_TOKEN_TTL = 86400
|
||||
|
||||
|
||||
class ZenmoneyError(RuntimeError):
|
||||
"""Upstream refused or misbehaved."""
|
||||
|
||||
|
||||
class ZenmoneyAuthError(ZenmoneyError):
|
||||
"""Credentials are missing, expired or rejected."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenBundle:
|
||||
access_token: str
|
||||
refresh_token: str | None = None
|
||||
expires_at: datetime | None = None
|
||||
|
||||
@property
|
||||
def expired(self) -> bool:
|
||||
if self.expires_at is None:
|
||||
return False
|
||||
return self.expires_at - EXPIRY_SKEW <= datetime.now(UTC)
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
return {
|
||||
"access_token": self.access_token,
|
||||
"refresh_token": self.refresh_token,
|
||||
"expires_at": self.expires_at.isoformat() if self.expires_at else None,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: dict[str, Any] | None) -> TokenBundle | None:
|
||||
if not payload or not payload.get("access_token"):
|
||||
return None
|
||||
raw_expires = payload.get("expires_at")
|
||||
expires_at: datetime | None = None
|
||||
if isinstance(raw_expires, str):
|
||||
try:
|
||||
expires_at = datetime.fromisoformat(raw_expires)
|
||||
except ValueError:
|
||||
expires_at = None
|
||||
if expires_at is not None and expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=UTC)
|
||||
return cls(
|
||||
access_token=str(payload["access_token"]),
|
||||
refresh_token=payload.get("refresh_token"),
|
||||
expires_at=expires_at,
|
||||
)
|
||||
|
||||
|
||||
class ZenmoneyClient:
|
||||
"""One diff call per sync, plus the token rotation around it.
|
||||
|
||||
Needs the sync session because in OAuth mode the rotated pair must be persisted in
|
||||
`source_credential` — the worker runs unattended, so a token it fetched and lost
|
||||
would strand the next run.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
http: httpx.AsyncClient | None = None,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
self._session = session
|
||||
self._http = http
|
||||
self._owns_http = http is None
|
||||
self._token: TokenBundle | None = None
|
||||
|
||||
# --- lifecycle ---------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def oauth_mode(self) -> bool:
|
||||
return bool(self._settings.zenmoney_client_id and self._settings.zenmoney_client_secret)
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
if self._http is None:
|
||||
# trust_env stays on: ZenMoney is reachable only through the host proxy
|
||||
self._http = httpx.AsyncClient(timeout=TIMEOUT)
|
||||
return self._http
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._http is not None and self._owns_http:
|
||||
await self._http.aclose()
|
||||
self._http = None
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
tb: TracebackType | None,
|
||||
) -> None:
|
||||
await self.aclose()
|
||||
|
||||
# --- diff --------------------------------------------------------------------
|
||||
|
||||
async def diff(self, server_timestamp: int) -> dict[str, Any]:
|
||||
body: dict[str, Any] = {
|
||||
"currentClientTimestamp": int(datetime.now(UTC).timestamp()),
|
||||
"serverTimestamp": server_timestamp,
|
||||
}
|
||||
if server_timestamp == 0:
|
||||
body["forceFetch"] = list(ENTITY_TYPES)
|
||||
|
||||
token = await self.access_token()
|
||||
resp = await self._post_diff(token, body)
|
||||
if resp.status_code == 401:
|
||||
if not self.oauth_mode:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZenMoney rejected the static access token (HTTP 401). ZenMoney access "
|
||||
"tokens live 24 hours — renew ZENMONEY_TOKEN (e.g. at zerro.app/token) "
|
||||
"and put the new value in .env, or configure ZENMONEY_CLIENT_ID / "
|
||||
"ZENMONEY_CLIENT_SECRET / ZENMONEY_REFRESH_TOKEN for automatic rotation."
|
||||
)
|
||||
log.info("zenmoney: diff got 401, refreshing the access token once")
|
||||
refreshed = await self._refresh_token()
|
||||
resp = await self._post_diff(refreshed.access_token, body)
|
||||
if resp.status_code == 401:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZenMoney rejected a freshly refreshed access token (HTTP 401); check "
|
||||
"ZENMONEY_CLIENT_ID / ZENMONEY_CLIENT_SECRET and the refresh token "
|
||||
"stored in source_credential."
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise ZenmoneyError(
|
||||
f"ZenMoney /v8/diff/ failed: HTTP {resp.status_code} {resp.text[:500]}"
|
||||
)
|
||||
data = json.loads(resp.text)
|
||||
if not isinstance(data, dict):
|
||||
raise ZenmoneyError("ZenMoney /v8/diff/ returned a non-object body")
|
||||
return data
|
||||
|
||||
async def _post_diff(self, token: str, body: dict[str, Any]) -> httpx.Response:
|
||||
return await self._client().post(
|
||||
DIFF_URL,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json=body,
|
||||
)
|
||||
|
||||
# --- tokens ------------------------------------------------------------------
|
||||
|
||||
async def access_token(self) -> str:
|
||||
if not self.oauth_mode:
|
||||
if not self._settings.zenmoney_token:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZENMONEY_TOKEN is not set — put a ZenMoney access token in .env "
|
||||
"(e.g. from zerro.app/token), or configure ZENMONEY_CLIENT_ID / "
|
||||
"ZENMONEY_CLIENT_SECRET / ZENMONEY_REFRESH_TOKEN for OAuth rotation."
|
||||
)
|
||||
return self._settings.zenmoney_token
|
||||
|
||||
bundle = self._token or await self._load_credential()
|
||||
if bundle is None or bundle.expired:
|
||||
bundle = await self._refresh_token(bundle)
|
||||
self._token = bundle
|
||||
return bundle.access_token
|
||||
|
||||
async def _load_credential(self) -> TokenBundle | None:
|
||||
row = await self._session.get(SourceCredential, SOURCE)
|
||||
return TokenBundle.from_payload(row.payload if row else None)
|
||||
|
||||
async def _refresh_token(self, bundle: TokenBundle | None = None) -> TokenBundle:
|
||||
settings = self._settings
|
||||
known = bundle or self._token or await self._load_credential()
|
||||
refresh_token = (known.refresh_token if known else None) or settings.zenmoney_refresh_token
|
||||
if not refresh_token:
|
||||
raise ZenmoneyAuthError(
|
||||
"OAuth mode is configured but no refresh token is available — set "
|
||||
"ZENMONEY_REFRESH_TOKEN in .env to seed source_credential."
|
||||
)
|
||||
resp = await self._client().post(
|
||||
TOKEN_URL,
|
||||
data={
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"client_id": settings.zenmoney_client_id or "",
|
||||
"client_secret": settings.zenmoney_client_secret or "",
|
||||
},
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZenMoney refused to refresh the access token: "
|
||||
f"HTTP {resp.status_code} {resp.text[:500]}"
|
||||
)
|
||||
data = resp.json()
|
||||
access_token = data.get("access_token")
|
||||
if not access_token:
|
||||
raise ZenmoneyAuthError("ZenMoney token response carried no access_token")
|
||||
try:
|
||||
ttl = int(data.get("expires_in") or DEFAULT_TOKEN_TTL)
|
||||
except (TypeError, ValueError):
|
||||
ttl = DEFAULT_TOKEN_TTL
|
||||
fresh = TokenBundle(
|
||||
access_token=str(access_token),
|
||||
refresh_token=data.get("refresh_token") or refresh_token,
|
||||
expires_at=datetime.now(UTC) + timedelta(seconds=ttl),
|
||||
)
|
||||
await self._store_credential(fresh)
|
||||
self._token = fresh
|
||||
return fresh
|
||||
|
||||
async def _store_credential(self, bundle: TokenBundle) -> None:
|
||||
payload = bundle.to_payload()
|
||||
stmt = pg_insert(SourceCredential).values(source=SOURCE, payload=payload)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["source"],
|
||||
set_={"payload": stmt.excluded.payload, "updated_at": datetime.now(UTC)},
|
||||
)
|
||||
await self._session.execute(stmt)
|
||||
# committed right away: a token we fetched but lost would strand the next run
|
||||
await self._session.commit()
|
||||
@@ -0,0 +1,483 @@
|
||||
"""Raw ZenMoney entities -> core tables (plan §1.2, §1.7).
|
||||
|
||||
The mapper never looks at the diff itself: it rebuilds the core rows from the full
|
||||
`raw_zenmoney_entity` table, so it is idempotent and self-healing (a mapping bug is fixed
|
||||
by re-running it, no re-download). Personal volumes are ~10^4 transactions, so a full
|
||||
upsert per sync is cheap; `sync.py` skips the mapper entirely when the diff was empty.
|
||||
|
||||
Rows deleted upstream are gone from the raw table, so they are never re-created here —
|
||||
the flags `sync.py` sets while processing `deletion` survive.
|
||||
|
||||
Money: every amount goes through `_money()` into `Decimal`. ZenMoney sends amounts as JSON
|
||||
numbers, so they arrive from JSONB as Python floats; `Decimal(str(value))` round-trips the
|
||||
printed (≤2 decimals) value exactly. Nothing float-valued is ever stored.
|
||||
|
||||
What is deliberately NOT touched here (owned by `analytics/classify.py`): `payee_canonical`,
|
||||
`is_one_off`, `trip_id`, and the refined `flow_type`/`category_id`. The mapper only lays
|
||||
down the base values ZenMoney itself implies.
|
||||
|
||||
Account fields the USER owns (`PATCH /accounts/{id}`): `name`, `role`, `include_in_net_worth`
|
||||
— plus `mirror_of_account_id` and `primary_event_source`, which the mapper never produces at
|
||||
all. They are seeded on INSERT and never overwritten on conflict, so a sync cannot undo an
|
||||
edit. The single exception: an account the source marks `archive` is forced out of net worth,
|
||||
the same way `sync.py` treats a deletion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from datetime import UTC, date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, delete, false, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.config import Settings
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
AccountKind,
|
||||
AccountRole,
|
||||
CashTxn,
|
||||
CashTxnTag,
|
||||
Category,
|
||||
FlowType,
|
||||
Merchant,
|
||||
RawZenmoneyEntity,
|
||||
)
|
||||
|
||||
SOURCE = "zenmoney"
|
||||
CHUNK = 500
|
||||
|
||||
KIND_BY_TYPE: dict[str, AccountKind] = {
|
||||
"cash": AccountKind.zm_cash,
|
||||
"ccard": AccountKind.zm_card,
|
||||
"checking": AccountKind.zm_checking,
|
||||
"deposit": AccountKind.zm_deposit,
|
||||
"loan": AccountKind.zm_loan,
|
||||
"emoney": AccountKind.zm_emoney,
|
||||
"debt": AccountKind.zm_debt,
|
||||
}
|
||||
"""ZenMoney `account.type` -> `AccountKind`. Only `ccard` -> `zm_card` is not a literal
|
||||
"zm_" + type: the enum in `models/accounts.py` spells the card kind without the extra c."""
|
||||
|
||||
DEBT_TYPES = frozenset({"loan", "debt"})
|
||||
DEPOSIT_FIELDS = (
|
||||
"capitalization",
|
||||
"percent",
|
||||
"startDate",
|
||||
"endDateOffset",
|
||||
"endDateOffsetInterval",
|
||||
"payoffStep",
|
||||
"payoffInterval",
|
||||
)
|
||||
META_FIELDS = ("reminderMarker", "latitude", "longitude")
|
||||
|
||||
|
||||
def _money(value: Any) -> Decimal | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, Decimal):
|
||||
return value
|
||||
try:
|
||||
return Decimal(str(value))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _int(value: Any) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _ccy(codes: dict[int, str], value: Any) -> str | None:
|
||||
"""Instrument id -> currency code. `None` for an absent or unknown instrument; id 0 is a
|
||||
perfectly valid key and must not collapse into a sentinel."""
|
||||
ident = _int(value)
|
||||
return codes.get(ident) if ident is not None else None
|
||||
|
||||
|
||||
def _day(value: Any) -> date | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _chunks(
|
||||
rows: Sequence[dict[str, Any]], size: int = CHUNK
|
||||
) -> Iterator[Sequence[dict[str, Any]]]:
|
||||
for start in range(0, len(rows), size):
|
||||
yield rows[start : start + size]
|
||||
|
||||
|
||||
async def _upsert(
|
||||
session: AsyncSession,
|
||||
model: type[Any],
|
||||
rows: Sequence[dict[str, Any]],
|
||||
conflict: list[str],
|
||||
update_cols: Iterable[str],
|
||||
set_extra: Callable[[Any], dict[str, Any]] | None = None,
|
||||
) -> int:
|
||||
"""Postgres INSERT … ON CONFLICT DO UPDATE, chunked. `update_cols` deliberately omits
|
||||
the columns other layers own, so re-mapping cannot clobber them. `set_extra` gets the
|
||||
insert statement and returns SET entries that are not a plain copy from `excluded`."""
|
||||
cols = list(update_cols)
|
||||
for chunk in _chunks(rows):
|
||||
stmt = pg_insert(model).values(list(chunk))
|
||||
set_: dict[str, Any] = {name: getattr(stmt.excluded, name) for name in cols}
|
||||
if set_extra is not None:
|
||||
set_.update(set_extra(stmt))
|
||||
stmt = stmt.on_conflict_do_update(index_elements=conflict, set_=set_)
|
||||
await session.execute(stmt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def _source_ids(session: AsyncSession, model: type[Any]) -> dict[str, int]:
|
||||
rows = await session.execute(select(model.source_id, model.id).where(model.source == SOURCE))
|
||||
return {source_id: row_id for source_id, row_id in rows.all()}
|
||||
|
||||
|
||||
async def load_raw(session: AsyncSession) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Every raw entity, grouped by type."""
|
||||
rows = await session.execute(select(RawZenmoneyEntity.entity_type, RawZenmoneyEntity.payload))
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
for entity_type, payload in rows.all():
|
||||
if isinstance(payload, dict):
|
||||
grouped.setdefault(entity_type, []).append(payload)
|
||||
return grouped
|
||||
|
||||
|
||||
def instrument_codes(instruments: list[dict[str, Any]]) -> dict[int, str]:
|
||||
"""ZenMoney instrument id -> 3-letter code ('RUB')."""
|
||||
codes: dict[int, str] = {}
|
||||
for payload in instruments:
|
||||
ident = _int(payload.get("id"))
|
||||
code = payload.get("shortTitle") or payload.get("title")
|
||||
if ident is not None and isinstance(code, str) and code:
|
||||
codes[ident] = code.upper()[:3]
|
||||
return codes
|
||||
|
||||
|
||||
async def map_all(session: AsyncSession, settings: Settings) -> tuple[dict[str, int], list[str]]:
|
||||
warnings: list[str] = []
|
||||
raw = await load_raw(session)
|
||||
codes = instrument_codes(raw.get("instrument", []))
|
||||
|
||||
categories = await _map_categories(session, raw.get("tag", []))
|
||||
merchants = await _map_merchants(session, raw.get("merchant", []))
|
||||
accounts = await _map_accounts(session, raw.get("account", []), codes, settings, warnings)
|
||||
transactions = await _map_transactions(
|
||||
session,
|
||||
raw.get("transaction", []),
|
||||
codes=codes,
|
||||
accounts=accounts,
|
||||
categories=categories,
|
||||
merchants=merchants,
|
||||
warnings=warnings,
|
||||
)
|
||||
counts = {
|
||||
"instruments": len(codes),
|
||||
"categories": len(categories),
|
||||
"merchants": len(merchants),
|
||||
"accounts": len(accounts),
|
||||
"transactions": transactions,
|
||||
}
|
||||
return counts, warnings
|
||||
|
||||
|
||||
# --- tags -> category -----------------------------------------------------------------
|
||||
|
||||
|
||||
async def _map_categories(session: AsyncSession, tags: list[dict[str, Any]]) -> dict[str, int]:
|
||||
"""Two passes: upsert the rows, then resolve `parent_id` (ZenMoney nests one level)."""
|
||||
rows = [
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": str(payload["id"]),
|
||||
"name": str(payload.get("title") or ""),
|
||||
"icon": payload.get("icon"),
|
||||
"color": _int(payload.get("color")),
|
||||
"show_income": bool(payload.get("showIncome", True)),
|
||||
"show_outcome": bool(payload.get("showOutcome", True)),
|
||||
"archived": False,
|
||||
}
|
||||
for payload in tags
|
||||
if payload.get("id") is not None
|
||||
]
|
||||
if rows:
|
||||
await _upsert(
|
||||
session,
|
||||
Category,
|
||||
rows,
|
||||
["source", "source_id"],
|
||||
("name", "icon", "color", "show_income", "show_outcome"),
|
||||
)
|
||||
ids = await _source_ids(session, Category)
|
||||
if not ids:
|
||||
return ids
|
||||
|
||||
parents = [
|
||||
{"id": ids[str(payload["id"])], "parent_id": ids.get(str(payload.get("parent")))}
|
||||
for payload in tags
|
||||
if payload.get("id") is not None and str(payload["id"]) in ids
|
||||
]
|
||||
if parents:
|
||||
await session.execute(update(Category), parents)
|
||||
return ids
|
||||
|
||||
|
||||
async def _map_merchants(session: AsyncSession, merchants: list[dict[str, Any]]) -> dict[str, int]:
|
||||
rows = [
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": str(payload["id"]),
|
||||
"name": str(payload.get("title") or ""),
|
||||
}
|
||||
for payload in merchants
|
||||
if payload.get("id") is not None
|
||||
]
|
||||
if rows:
|
||||
await _upsert(session, Merchant, rows, ["source", "source_id"], ("name",))
|
||||
return await _source_ids(session, Merchant)
|
||||
|
||||
|
||||
# --- accounts -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def account_role(acc_type: str, savings: bool) -> AccountRole:
|
||||
"""loan/debt are what we owe; an explicit savings flag or a deposit is savings money;
|
||||
everything else is spendable. `investment` is left to broker accounts (phase 2), and
|
||||
`mirror_of_account_id` / `primary_event_source` stay whatever the user set."""
|
||||
if acc_type in DEBT_TYPES:
|
||||
return AccountRole.debt
|
||||
if savings or acc_type == "deposit":
|
||||
return AccountRole.savings
|
||||
return AccountRole.liquid
|
||||
|
||||
|
||||
async def _map_accounts(
|
||||
session: AsyncSession,
|
||||
accounts: list[dict[str, Any]],
|
||||
codes: dict[int, str],
|
||||
settings: Settings,
|
||||
warnings: list[str],
|
||||
) -> dict[str, int]:
|
||||
observed_at = datetime.now(UTC)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for payload in accounts:
|
||||
if payload.get("id") is None:
|
||||
continue
|
||||
source_id = str(payload["id"])
|
||||
acc_type = str(payload.get("type") or "cash")
|
||||
kind = KIND_BY_TYPE.get(acc_type)
|
||||
if kind is None:
|
||||
warnings.append(
|
||||
f"account {source_id}: unknown ZenMoney type {acc_type!r}, stored as zm_cash"
|
||||
)
|
||||
kind = AccountKind.zm_cash
|
||||
currency = _ccy(codes, payload.get("instrument"))
|
||||
if not currency:
|
||||
currency = settings.base_currency
|
||||
warnings.append(
|
||||
f"account {source_id}: instrument {payload.get('instrument')!r} is unknown, "
|
||||
f"currency assumed {currency}"
|
||||
)
|
||||
archived = bool(payload.get("archive"))
|
||||
terms = {
|
||||
field: payload[field] for field in DEPOSIT_FIELDS if payload.get(field) is not None
|
||||
}
|
||||
rows.append(
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": source_id,
|
||||
"kind": kind,
|
||||
# name / role / include_in_net_worth are seeded here and then owned by the
|
||||
# user: they are absent from the ON CONFLICT set below
|
||||
"name": str(payload.get("title") or source_id),
|
||||
"currency": currency,
|
||||
"role": account_role(acc_type, bool(payload.get("savings"))),
|
||||
# inBalance is ZenMoney's own "counts towards my balance" switch
|
||||
"include_in_net_worth": bool(payload.get("inBalance", True)) and not archived,
|
||||
"archived": archived,
|
||||
"opened_at": _day(payload.get("startDate")),
|
||||
"deposit_terms": terms or None,
|
||||
"balance": _money(payload.get("balance")),
|
||||
"start_balance": _money(payload.get("startBalance")),
|
||||
"credit_limit": _money(payload.get("creditLimit")),
|
||||
"balance_as_of": observed_at,
|
||||
}
|
||||
)
|
||||
if rows:
|
||||
await _upsert(
|
||||
session,
|
||||
Account,
|
||||
rows,
|
||||
["source", "source_id"],
|
||||
(
|
||||
"kind",
|
||||
"currency",
|
||||
"archived",
|
||||
"opened_at",
|
||||
"deposit_terms",
|
||||
"balance",
|
||||
"start_balance",
|
||||
"credit_limit",
|
||||
"balance_as_of",
|
||||
),
|
||||
# archived upstream still forces the account out of net worth (same as a
|
||||
# deletion); otherwise the user's own switch wins
|
||||
set_extra=lambda stmt: {
|
||||
"include_in_net_worth": case(
|
||||
(stmt.excluded.archived, false()),
|
||||
else_=Account.include_in_net_worth,
|
||||
)
|
||||
},
|
||||
)
|
||||
return await _source_ids(session, Account)
|
||||
|
||||
|
||||
# --- transactions ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def flow_type(income: Decimal, outcome: Decimal, deleted: bool) -> FlowType:
|
||||
"""The base classification ZenMoney itself implies. Rules (savings_transfer, one_off,
|
||||
broker flows) refine it later in `analytics/classify.py`."""
|
||||
if deleted:
|
||||
return FlowType.deleted
|
||||
if income > 0 and outcome > 0:
|
||||
return FlowType.internal_transfer
|
||||
if outcome > 0:
|
||||
return FlowType.expense
|
||||
if income > 0:
|
||||
return FlowType.income
|
||||
return FlowType.other
|
||||
|
||||
|
||||
async def _map_transactions(
|
||||
session: AsyncSession,
|
||||
transactions: list[dict[str, Any]],
|
||||
*,
|
||||
codes: dict[int, str],
|
||||
accounts: dict[str, int],
|
||||
categories: dict[str, int],
|
||||
merchants: dict[str, int],
|
||||
warnings: list[str],
|
||||
) -> int:
|
||||
rows: list[dict[str, Any]] = []
|
||||
tags_by_source_id: dict[str, list[int]] = {}
|
||||
for payload in transactions:
|
||||
if payload.get("id") is None:
|
||||
continue
|
||||
source_id = str(payload["id"])
|
||||
day = _day(payload.get("date"))
|
||||
if day is None:
|
||||
warnings.append(f"transaction {source_id}: unparsable date {payload.get('date')!r}")
|
||||
continue
|
||||
created = _int(payload.get("created")) or _int(payload.get("changed"))
|
||||
ts = (
|
||||
datetime.fromtimestamp(created, UTC)
|
||||
if created is not None
|
||||
else datetime.combine(day, datetime.min.time(), UTC)
|
||||
)
|
||||
income = _money(payload.get("income")) or Decimal(0)
|
||||
outcome = _money(payload.get("outcome")) or Decimal(0)
|
||||
deleted = bool(payload.get("deleted"))
|
||||
|
||||
tag_ids = [
|
||||
categories[str(tag)] for tag in (payload.get("tag") or []) if str(tag) in categories
|
||||
]
|
||||
tags_by_source_id[source_id] = tag_ids
|
||||
primary_category_id = tag_ids[0] if tag_ids else None
|
||||
meta = {field: payload[field] for field in META_FIELDS if payload.get(field) is not None}
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": source_id,
|
||||
"ts": ts,
|
||||
"date": day,
|
||||
"income": income,
|
||||
"income_currency": _ccy(codes, payload.get("incomeInstrument")),
|
||||
"income_account_id": accounts.get(str(payload.get("incomeAccount"))),
|
||||
"outcome": outcome,
|
||||
"outcome_currency": _ccy(codes, payload.get("outcomeInstrument")),
|
||||
"outcome_account_id": accounts.get(str(payload.get("outcomeAccount"))),
|
||||
"op_income": _money(payload.get("opIncome")),
|
||||
"op_income_currency": _ccy(codes, payload.get("opIncomeInstrument")),
|
||||
"op_outcome": _money(payload.get("opOutcome")),
|
||||
"op_outcome_currency": _ccy(codes, payload.get("opOutcomeInstrument")),
|
||||
"payee": payload.get("payee"),
|
||||
"original_payee": payload.get("originalPayee"),
|
||||
"merchant_id": merchants.get(str(payload.get("merchant"))),
|
||||
"comment": payload.get("comment"),
|
||||
"mcc": _int(payload.get("mcc")),
|
||||
"hold": bool(payload.get("hold")),
|
||||
"deleted": deleted,
|
||||
"changed": _int(payload.get("changed")),
|
||||
"primary_category_id": primary_category_id,
|
||||
"flow_type": flow_type(income, outcome, deleted),
|
||||
"category_id": primary_category_id,
|
||||
"is_one_off": False,
|
||||
}
|
||||
| {"meta": meta or None}
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
await _upsert(
|
||||
session,
|
||||
CashTxn,
|
||||
rows,
|
||||
["source", "source_id"],
|
||||
(
|
||||
"ts",
|
||||
"date",
|
||||
"income",
|
||||
"income_currency",
|
||||
"income_account_id",
|
||||
"outcome",
|
||||
"outcome_currency",
|
||||
"outcome_account_id",
|
||||
"op_income",
|
||||
"op_income_currency",
|
||||
"op_outcome",
|
||||
"op_outcome_currency",
|
||||
"payee",
|
||||
"original_payee",
|
||||
"merchant_id",
|
||||
"comment",
|
||||
"mcc",
|
||||
"hold",
|
||||
"deleted",
|
||||
"changed",
|
||||
"primary_category_id",
|
||||
"flow_type",
|
||||
"category_id",
|
||||
"meta",
|
||||
),
|
||||
)
|
||||
|
||||
txn_ids = await _source_ids(session, CashTxn)
|
||||
links: list[dict[str, Any]] = []
|
||||
for source_id, tag_ids in tags_by_source_id.items():
|
||||
txn_id = txn_ids.get(source_id)
|
||||
if txn_id is None:
|
||||
continue
|
||||
links.extend(
|
||||
{"txn_id": txn_id, "ord": ord_, "category_id": category_id}
|
||||
for ord_, category_id in enumerate(tag_ids)
|
||||
)
|
||||
mapped_ids = [txn_ids[s] for s in tags_by_source_id if s in txn_ids]
|
||||
if mapped_ids:
|
||||
await session.execute(delete(CashTxnTag).where(CashTxnTag.txn_id.in_(mapped_ids)))
|
||||
for chunk in _chunks(links):
|
||||
await session.execute(pg_insert(CashTxnTag).values(list(chunk)))
|
||||
return len(rows)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""The `zenmoney` source: one /v8/diff/ call per run, raw tier, then the core mapping.
|
||||
|
||||
Algorithm (plan §1.7):
|
||||
|
||||
1. POST /v8/diff/ with the stored cursor (`serverTimestamp`; `forceFetch` when it is 0).
|
||||
2. Upsert every returned entity into `raw_zenmoney_entity` by (entity_type, id).
|
||||
3. Apply `deletion`: drop the raw row, remember the fact in `raw_zenmoney_deletion`, and
|
||||
flag the core row — a deleted transaction becomes `deleted` (flow_type `deleted`), a
|
||||
deleted account or tag becomes archived.
|
||||
4. Re-map the raw tables into the core tables (see `mapper.py`) — always from the full raw
|
||||
tier, never from the diff alone, so the result does not depend on how the history was
|
||||
downloaded.
|
||||
5. Store the new `serverTimestamp` as the cursor.
|
||||
|
||||
Re-runs are cheap and inert: an empty diff (no entities, no deletions) skips the mapper
|
||||
altogether and reports `changed=False`, so the worker also skips the metrics refresh.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
CashTxn,
|
||||
Category,
|
||||
FlowType,
|
||||
RawZenmoneyDeletion,
|
||||
RawZenmoneyEntity,
|
||||
)
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.zenmoney import mapper
|
||||
from fintracker.sources.zenmoney.client import ENTITY_TYPES, SOURCE, ZenmoneyClient
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZenmoneySource:
|
||||
name = SOURCE
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
cursor_before = _as_timestamp(ctx.cursor_before)
|
||||
async with ZenmoneyClient(ctx.settings, ctx.session) as client:
|
||||
data = await client.diff(cursor_before)
|
||||
|
||||
raw_upserted = await _upsert_entities(ctx, data)
|
||||
deleted = await _apply_deletions(ctx, data)
|
||||
changed = bool(raw_upserted or deleted)
|
||||
|
||||
counts: dict[str, int] = {"raw_upserted": raw_upserted, "deleted": deleted}
|
||||
warnings: list[str] = []
|
||||
if changed:
|
||||
mapped, warnings = await mapper.map_all(ctx.session, ctx.settings)
|
||||
counts.update(mapped)
|
||||
await ctx.session.commit()
|
||||
|
||||
cursor_after = data.get("serverTimestamp")
|
||||
log.info(
|
||||
"zenmoney: %s raw rows, %s deletions, cursor %s -> %s",
|
||||
raw_upserted,
|
||||
deleted,
|
||||
cursor_before,
|
||||
cursor_after,
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=str(int(cursor_after)) if cursor_after is not None else None,
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=changed,
|
||||
)
|
||||
|
||||
|
||||
def _as_timestamp(cursor: str | None) -> int:
|
||||
if not cursor:
|
||||
return 0
|
||||
try:
|
||||
return int(cursor)
|
||||
except ValueError:
|
||||
log.warning("zenmoney: unusable cursor %r, doing a full pull", cursor)
|
||||
return 0
|
||||
|
||||
|
||||
async def _upsert_entities(ctx: SyncContext, data: dict[str, Any]) -> int:
|
||||
ingested_at = datetime.now(UTC)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for entity_type in ENTITY_TYPES:
|
||||
for payload in data.get(entity_type) or []:
|
||||
if not isinstance(payload, dict) or payload.get("id") is None:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"entity_type": entity_type,
|
||||
"id": str(payload["id"]),
|
||||
"changed": payload.get("changed"),
|
||||
"payload": payload,
|
||||
"ingested_at": ingested_at,
|
||||
}
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
for start in range(0, len(rows), mapper.CHUNK):
|
||||
chunk = rows[start : start + mapper.CHUNK]
|
||||
stmt = pg_insert(RawZenmoneyEntity).values(chunk)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["entity_type", "id"],
|
||||
set_={
|
||||
"changed": stmt.excluded.changed,
|
||||
"payload": stmt.excluded.payload,
|
||||
"ingested_at": stmt.excluded.ingested_at,
|
||||
},
|
||||
)
|
||||
await ctx.session.execute(stmt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def _apply_deletions(ctx: SyncContext, data: dict[str, Any]) -> int:
|
||||
deletions = [d for d in (data.get("deletion") or []) if isinstance(d, dict)]
|
||||
if not deletions:
|
||||
return 0
|
||||
session = ctx.session
|
||||
count = 0
|
||||
for item in deletions:
|
||||
entity_type = str(item.get("object") or "")
|
||||
entity_id = str(item.get("id") or "")
|
||||
if not entity_type or not entity_id:
|
||||
continue
|
||||
await session.execute(
|
||||
delete(RawZenmoneyEntity).where(
|
||||
RawZenmoneyEntity.entity_type == entity_type,
|
||||
RawZenmoneyEntity.id == entity_id,
|
||||
)
|
||||
)
|
||||
stmt = pg_insert(RawZenmoneyDeletion).values(
|
||||
entity_type=entity_type,
|
||||
id=entity_id,
|
||||
stamp=item.get("stamp"),
|
||||
deleted_at=datetime.now(UTC),
|
||||
)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["entity_type", "id"],
|
||||
set_={"stamp": stmt.excluded.stamp, "deleted_at": stmt.excluded.deleted_at},
|
||||
)
|
||||
await session.execute(stmt)
|
||||
|
||||
if entity_type == "transaction":
|
||||
await session.execute(
|
||||
update(CashTxn)
|
||||
.where(CashTxn.source == SOURCE, CashTxn.source_id == entity_id)
|
||||
.values(deleted=True, flow_type=FlowType.deleted)
|
||||
)
|
||||
elif entity_type == "account":
|
||||
await session.execute(
|
||||
update(Account)
|
||||
.where(Account.source == SOURCE, Account.source_id == entity_id)
|
||||
.values(archived=True, include_in_net_worth=False)
|
||||
)
|
||||
elif entity_type == "tag":
|
||||
await session.execute(
|
||||
update(Category)
|
||||
.where(Category.source == SOURCE, Category.source_id == entity_id)
|
||||
.values(archived=True)
|
||||
)
|
||||
count += 1
|
||||
return count
|
||||
Reference in New Issue
Block a user