feat(sources): контракт источников, worker и синк ZenMoney + ЦБ
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
This commit is contained in:
Dmitry
@@ -0,0 +1,168 @@
|
||||
"""FX rates. Later phases add price_daily / price_last / corporate_action here (plan §1.6)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class RawCbrRate(Base):
|
||||
"""CBR official rate as published: RUB per `nominal` units of `ccy`, on business days only."""
|
||||
|
||||
__tablename__ = "raw_cbr_rate"
|
||||
|
||||
rate_date: Mapped[date] = mapped_column(primary_key=True)
|
||||
ccy: Mapped[str] = mapped_column(String(3), primary_key=True)
|
||||
nominal: Mapped[int] = mapped_column(Integer, default=1)
|
||||
value: Mapped[Decimal]
|
||||
"""RUB per `nominal` units, exactly as CBR printed it."""
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class FxRateDaily(Base):
|
||||
"""Every calendar day has a rate: business-day quotes carried forward (and backward before
|
||||
the first quote). RUB is present as 1.0. Rebuilt by pricing/fx.py on each refresh."""
|
||||
|
||||
__tablename__ = "fx_rate_daily"
|
||||
|
||||
d: Mapped[date] = mapped_column(primary_key=True)
|
||||
ccy: Mapped[str] = mapped_column(String(3), primary_key=True)
|
||||
rate_rub: Mapped[Decimal]
|
||||
"""RUB per ONE unit of ccy."""
|
||||
source: Mapped[str] = mapped_column(String(16), default="cbr")
|
||||
is_carried: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class PriceDaily(Base):
|
||||
"""Daily close per instrument. Bonds quote in percent of nominal, so `price_pct` carries
|
||||
the quote as published and `close` is the money value it resolves to."""
|
||||
|
||||
__tablename__ = "price_daily"
|
||||
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
d: Mapped[date] = mapped_column(primary_key=True)
|
||||
close: Mapped[Decimal]
|
||||
open: Mapped[Decimal | None]
|
||||
high: Mapped[Decimal | None]
|
||||
low: Mapped[Decimal | None]
|
||||
volume: Mapped[Decimal | None]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
"""tinvest | moex | manual"""
|
||||
price_pct: Mapped[Decimal | None]
|
||||
"""Bond quote in percent of nominal, as the exchange published it."""
|
||||
accrued_interest: Mapped[Decimal | None]
|
||||
|
||||
|
||||
class PriceLast(Base):
|
||||
"""Latest known price per instrument — what the positions screen shows between refreshes."""
|
||||
|
||||
__tablename__ = "price_last"
|
||||
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
ts: Mapped[datetime]
|
||||
price: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
|
||||
|
||||
class PriceManual(Base):
|
||||
"""User-set prices for what no exchange quotes: real estate, crypto, custom holdings."""
|
||||
|
||||
__tablename__ = "price_manual"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
d: Mapped[date]
|
||||
price: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
note: Mapped[str | None] = mapped_column(String(256))
|
||||
|
||||
|
||||
class CorporateActionKind(enum.StrEnum):
|
||||
dividend = "dividend"
|
||||
coupon = "coupon"
|
||||
amortization = "amortization"
|
||||
repayment = "repayment"
|
||||
stock_split = "split"
|
||||
offer = "offer"
|
||||
|
||||
|
||||
class CorporateActionStatus(enum.StrEnum):
|
||||
forecast = "forecast"
|
||||
"""Projected from history — never mixed with announced money in the paid totals."""
|
||||
announced = "announced"
|
||||
paid = "paid"
|
||||
cancelled = "cancelled"
|
||||
|
||||
|
||||
class CorporateAction(TimestampMixin, Base):
|
||||
"""Declared and projected payouts, the source of the dividend/coupon calendar."""
|
||||
|
||||
__tablename__ = "corporate_action"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("instrument_id", "kind", "source", "source_id"),
|
||||
Index("ix_corporate_action_pay_date", "pay_date"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
kind: Mapped[CorporateActionKind] = mapped_column(
|
||||
db_enum(CorporateActionKind, "corporate_action_kind")
|
||||
)
|
||||
status: Mapped[CorporateActionStatus] = mapped_column(
|
||||
db_enum(CorporateActionStatus, "corporate_action_status")
|
||||
)
|
||||
record_date: Mapped[date | None]
|
||||
ex_date: Mapped[date | None]
|
||||
pay_date: Mapped[date | None]
|
||||
amount_per_unit: Mapped[Decimal | None]
|
||||
currency: Mapped[str | None] = mapped_column(String(3))
|
||||
ratio: Mapped[Decimal | None]
|
||||
"""Split ratio; NULL for cash events."""
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
source_id: Mapped[str | None] = mapped_column(String(128))
|
||||
|
||||
|
||||
class PositionSnapshot(Base):
|
||||
"""What the broker says a position is — kept for reconciliation, never for analytics."""
|
||||
|
||||
__tablename__ = "position_snapshot"
|
||||
|
||||
account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
as_of: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
qty: Mapped[Decimal]
|
||||
avg_price: Mapped[Decimal | None]
|
||||
market_value: Mapped[Decimal | None]
|
||||
currency: Mapped[str | None] = mapped_column(String(3))
|
||||
|
||||
|
||||
class CashSnapshot(Base):
|
||||
"""Broker-reported cash per currency — the counterpart of `position_snapshot`."""
|
||||
|
||||
__tablename__ = "cash_snapshot"
|
||||
|
||||
account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
currency: Mapped[str] = mapped_column(String(3), primary_key=True)
|
||||
as_of: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
balance: Mapped[Decimal]
|
||||
blocked: Mapped[Decimal | None]
|
||||
@@ -0,0 +1,204 @@
|
||||
"""ZenMoney side: raw diff payloads, categories (tags), merchants, everyday transactions,
|
||||
hand-written rules and trips (plan §1.7)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class RawZenmoneyEntity(Base):
|
||||
"""Untouched JSON of every entity the /v8/diff endpoint returned; upserted by (type, id)."""
|
||||
|
||||
__tablename__ = "raw_zenmoney_entity"
|
||||
|
||||
entity_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
changed: Mapped[int | None] = mapped_column(BigInteger)
|
||||
payload: Mapped[dict[str, Any]]
|
||||
ingested_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawZenmoneyDeletion(Base):
|
||||
"""Deletions reported by diff; the raw row is removed, this keeps the fact for audit."""
|
||||
|
||||
__tablename__ = "raw_zenmoney_deletion"
|
||||
|
||||
entity_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
stamp: Mapped[int | None] = mapped_column(BigInteger)
|
||||
deleted_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class Category(TimestampMixin, Base):
|
||||
"""ZenMoney tag tree (max one level of nesting on their side)."""
|
||||
|
||||
__tablename__ = "category"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
parent_id: Mapped[int | None] = mapped_column(ForeignKey("category.id", ondelete="SET NULL"))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
icon: Mapped[str | None] = mapped_column(String(64))
|
||||
color: Mapped[int | None] = mapped_column(BigInteger)
|
||||
show_income: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
show_outcome: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
archived: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class Merchant(Base):
|
||||
__tablename__ = "merchant"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
|
||||
|
||||
class FlowType(enum.StrEnum):
|
||||
income = "income"
|
||||
expense = "expense"
|
||||
internal_transfer = "internal_transfer"
|
||||
savings_transfer = "savings_transfer"
|
||||
broker_external_flow = "broker_external_flow"
|
||||
deleted = "deleted"
|
||||
other = "other"
|
||||
|
||||
|
||||
class CashTxn(TimestampMixin, Base):
|
||||
"""One ZenMoney transaction. A transfer between own accounts is ONE row with both
|
||||
income and outcome > 0. Native amounts only — conversion happens in analytics."""
|
||||
|
||||
__tablename__ = "cash_txn"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
ts: Mapped[datetime]
|
||||
"""created timestamp from ZenMoney (UTC)."""
|
||||
date: Mapped[date] = mapped_column(index=True)
|
||||
income: Mapped[Decimal] = mapped_column(default=Decimal(0))
|
||||
income_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
income_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
outcome: Mapped[Decimal] = mapped_column(default=Decimal(0))
|
||||
outcome_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
outcome_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
op_income: Mapped[Decimal | None]
|
||||
op_income_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
op_outcome: Mapped[Decimal | None]
|
||||
op_outcome_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
payee: Mapped[str | None] = mapped_column(String(512))
|
||||
original_payee: Mapped[str | None] = mapped_column(String(512))
|
||||
merchant_id: Mapped[int | None] = mapped_column(ForeignKey("merchant.id", ondelete="SET NULL"))
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
mcc: Mapped[int | None] = mapped_column(Integer)
|
||||
hold: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
deleted: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
changed: Mapped[int | None] = mapped_column(BigInteger)
|
||||
primary_category_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("category.id", ondelete="SET NULL")
|
||||
)
|
||||
"""First tag as ZenMoney sent it (raw category)."""
|
||||
|
||||
# --- derived by analytics/classify (rules applied); rebuilt on every refresh ---
|
||||
flow_type: Mapped[FlowType] = mapped_column(
|
||||
db_enum(FlowType, "flow_type"), default=FlowType.other
|
||||
)
|
||||
category_id: Mapped[int | None] = mapped_column(ForeignKey("category.id", ondelete="SET NULL"))
|
||||
"""Effective category after rules (defaults to primary_category_id)."""
|
||||
payee_canonical: Mapped[str | None] = mapped_column(String(512))
|
||||
is_one_off: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
trip_id: Mapped[int | None] = mapped_column(ForeignKey("trip.id", ondelete="SET NULL"))
|
||||
meta: Mapped[dict[str, Any] | None]
|
||||
|
||||
|
||||
class CashTxnTag(Base):
|
||||
"""All tags of a transaction in ZenMoney order (ord 0 == primary)."""
|
||||
|
||||
__tablename__ = "cash_txn_tag"
|
||||
|
||||
txn_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("cash_txn.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
ord: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
category_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("category.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class RuleKind(enum.StrEnum):
|
||||
savings = "savings"
|
||||
"""expense -> savings_transfer (money moved into savings, not consumed)"""
|
||||
one_off = "one_off"
|
||||
"""still an expense, but out of the baseline and runway divisor"""
|
||||
category = "category"
|
||||
"""override category; `value` = category name"""
|
||||
payee = "payee"
|
||||
"""canonical payee name; `value` = the name"""
|
||||
broker_target = "broker_target"
|
||||
"""outflow funds a brokerage account; `value` = broker account id (str)"""
|
||||
ignore = "ignore"
|
||||
"""exclude from every metric: `flow_type` becomes `other`, and rule application STOPS for
|
||||
that transaction — no later rule (by priority) can put it back into a counted flow"""
|
||||
|
||||
|
||||
class RuleMatchType(enum.StrEnum):
|
||||
id = "id"
|
||||
payee = "payee"
|
||||
comment = "comment"
|
||||
category = "category"
|
||||
mcc = "mcc"
|
||||
account = "account"
|
||||
|
||||
|
||||
class Rule(TimestampMixin, Base):
|
||||
"""The judgement layer: ZenMoney knows what an operation *was*, rules say what it *meant*.
|
||||
`pattern` is matched case-insensitively with SQL LIKE semantics (`%` wildcards) except for
|
||||
`id`, `mcc` and `account`, which compare exactly."""
|
||||
|
||||
__tablename__ = "rule"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[RuleKind] = mapped_column(db_enum(RuleKind, "rule_kind"))
|
||||
match_type: Mapped[RuleMatchType] = mapped_column(db_enum(RuleMatchType, "rule_match_type"))
|
||||
pattern: Mapped[str] = mapped_column(String(512))
|
||||
value: Mapped[str | None] = mapped_column(String(512))
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
priority: Mapped[int] = mapped_column(Integer, default=100)
|
||||
last_matched_at: Mapped[datetime | None]
|
||||
match_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class Trip(TimestampMixin, Base):
|
||||
__tablename__ = "trip"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
date_from: Mapped[date]
|
||||
date_to: Mapped[date]
|
||||
country: Mapped[str | None] = mapped_column(String(2))
|
||||
@@ -0,0 +1,13 @@
|
||||
"""Data sources.
|
||||
|
||||
Importing this package pulls in every source package, and each of those calls
|
||||
`registry.register(...)` at import time — so the API, the worker and the CLI all see the
|
||||
same set of sources just by importing `fintracker.sources`.
|
||||
|
||||
`sources/base.py` and `sources/registry.py` must never import the source packages back:
|
||||
that is what keeps this import graph acyclic.
|
||||
"""
|
||||
|
||||
from fintracker.sources import cbr, moex, tinvest, zenmoney
|
||||
|
||||
__all__ = ["cbr", "moex", "tinvest", "zenmoney"]
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Contract every data source implements (plan §2.3).
|
||||
|
||||
A source pulls from one upstream (ZenMoney, T-Invest, MOEX, CBR) into the raw tier and
|
||||
optionally maps into core tables. It owns its cursor via `SyncContext` and reports what
|
||||
it did in `SyncResult`; the worker handles locking, run logging and scheduling.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.config import Settings
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncResult:
|
||||
cursor_after: str | None = None
|
||||
counts: dict[str, int] = field(default_factory=dict)
|
||||
warnings: list[str] = field(default_factory=list)
|
||||
changed: bool = True
|
||||
"""False when nothing new landed — lets the worker skip the metrics refresh."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncContext:
|
||||
session: AsyncSession
|
||||
settings: Settings
|
||||
cursor_before: str | None
|
||||
triggered_by: str
|
||||
extra: dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
class Source(Protocol):
|
||||
name: str
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult: ...
|
||||
@@ -0,0 +1,8 @@
|
||||
"""CBR FX source package. Importing it registers the source."""
|
||||
|
||||
from fintracker.sources.cbr.sync import CbrSource
|
||||
from fintracker.sources.registry import register
|
||||
|
||||
register(CbrSource())
|
||||
|
||||
__all__ = ["CbrSource"]
|
||||
@@ -0,0 +1,163 @@
|
||||
"""CBR (Bank of Russia) official FX rates over the public XML scripts.
|
||||
|
||||
Two endpoints:
|
||||
|
||||
* `GET /scripts/XML_daily.asp?date_req=DD/MM/YYYY` — every quoted currency on one day;
|
||||
used only for the `CharCode -> internal id` map ('USD' -> 'R01235').
|
||||
* `GET /scripts/XML_dynamic.asp?date_req1=…&date_req2=…&VAL_NM_RQ=R01235` — one currency
|
||||
over a date range, `<Record Date="01.09.2026"><Nominal>1</Nominal><Value>92,1234</Value>`.
|
||||
|
||||
Both answer windows-1251 XML with a decimal comma, and quote RUB per `Nominal` units
|
||||
(100 for JPY, 10 for CNY at times). Values are parsed into `Decimal` and kept exactly as
|
||||
printed, nominal included — dividing per unit happens later in `pricing/fx.py`.
|
||||
|
||||
NETWORK NOTE: `trust_env=False`. This host exports http_proxy/https_proxy/all_proxy, and
|
||||
cbr.ru fails the TLS handshake through that proxy, so the CBR client must bypass the
|
||||
environment entirely. (ZenMoney is the opposite case and keeps the proxy — see
|
||||
`sources/zenmoney/client.py`.)
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from datetime import date, datetime, timedelta
|
||||
from decimal import Decimal, InvalidOperation
|
||||
|
||||
import httpx
|
||||
|
||||
DAILY_URL = "https://www.cbr.ru/scripts/XML_daily.asp"
|
||||
DYNAMIC_URL = "https://www.cbr.ru/scripts/XML_dynamic.asp"
|
||||
ENCODING = "windows-1251"
|
||||
TIMEOUT = 60.0
|
||||
MAX_RANGE_DAYS = 366
|
||||
"""CBR copes with longer ranges, but requests are chunked to at most a year to be polite."""
|
||||
|
||||
|
||||
class CbrError(RuntimeError):
|
||||
"""CBR refused or returned something unparsable."""
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class CbrQuote:
|
||||
rate_date: date
|
||||
nominal: int
|
||||
value: Decimal
|
||||
"""RUB per `nominal` units, exactly as CBR printed it."""
|
||||
|
||||
|
||||
def new_http_client() -> httpx.AsyncClient:
|
||||
# trust_env=False: see NETWORK NOTE above
|
||||
return httpx.AsyncClient(trust_env=False, timeout=TIMEOUT)
|
||||
|
||||
|
||||
def _fmt(day: date) -> str:
|
||||
return day.strftime("%d/%m/%Y")
|
||||
|
||||
|
||||
def _decimal(text: str | None) -> Decimal | None:
|
||||
if not text:
|
||||
return None
|
||||
try:
|
||||
return Decimal(text.replace("\xa0", "").replace(" ", "").replace(",", "."))
|
||||
except InvalidOperation:
|
||||
return None
|
||||
|
||||
|
||||
def _parse_xml(payload: bytes) -> ET.Element:
|
||||
try:
|
||||
return ET.fromstring(payload.decode(ENCODING))
|
||||
except (ET.ParseError, UnicodeDecodeError) as exc:
|
||||
raise CbrError(f"CBR returned unparsable XML: {exc}") from exc
|
||||
|
||||
|
||||
def parse_code_map(payload: bytes) -> dict[str, str]:
|
||||
"""XML_daily -> {'USD': 'R01235', …}."""
|
||||
root = _parse_xml(payload)
|
||||
codes: dict[str, str] = {}
|
||||
for node in root.iter("Valute"):
|
||||
char_code = (node.findtext("CharCode") or "").strip().upper()
|
||||
cbr_id = (node.get("ID") or "").strip()
|
||||
if char_code and cbr_id:
|
||||
codes[char_code] = cbr_id
|
||||
return codes
|
||||
|
||||
|
||||
def parse_dynamic(payload: bytes) -> list[CbrQuote]:
|
||||
"""XML_dynamic -> quotes, nominal and value untouched."""
|
||||
root = _parse_xml(payload)
|
||||
quotes: list[CbrQuote] = []
|
||||
for node in root.iter("Record"):
|
||||
raw_date = node.get("Date")
|
||||
value = _decimal(node.findtext("Value"))
|
||||
if not raw_date or value is None:
|
||||
continue
|
||||
try:
|
||||
rate_date = datetime.strptime(raw_date, "%d.%m.%Y").date()
|
||||
except ValueError:
|
||||
continue
|
||||
nominal_text = (node.findtext("Nominal") or "1").strip()
|
||||
try:
|
||||
nominal = int(nominal_text.replace("\xa0", "").replace(" ", "")) or 1
|
||||
except ValueError:
|
||||
nominal = 1
|
||||
quotes.append(CbrQuote(rate_date=rate_date, nominal=nominal, value=value))
|
||||
return quotes
|
||||
|
||||
|
||||
def split_range(start: date, end: date, max_days: int = MAX_RANGE_DAYS) -> list[tuple[date, date]]:
|
||||
"""Chunk a long range into windows of at most `max_days` days."""
|
||||
if start > end:
|
||||
return []
|
||||
windows: list[tuple[date, date]] = []
|
||||
window_start = start
|
||||
while window_start <= end:
|
||||
window_end = min(end, window_start + timedelta(days=max_days - 1))
|
||||
windows.append((window_start, window_end))
|
||||
window_start = window_end + timedelta(days=1)
|
||||
return windows
|
||||
|
||||
|
||||
class CbrClient:
|
||||
def __init__(self, http: httpx.AsyncClient | None = None) -> None:
|
||||
self._http = http
|
||||
self._owns_http = http is None
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
if self._http is None:
|
||||
self._http = new_http_client()
|
||||
return self._http
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._http is not None and self._owns_http:
|
||||
await self._http.aclose()
|
||||
self._http = None
|
||||
|
||||
async def __aenter__(self) -> CbrClient:
|
||||
return self
|
||||
|
||||
async def __aexit__(self, *exc_info: object) -> None:
|
||||
await self.aclose()
|
||||
|
||||
async def _get(self, url: str, params: dict[str, str]) -> bytes:
|
||||
resp = await self._client().get(url, params=params)
|
||||
if resp.status_code >= 400:
|
||||
raise CbrError(f"{url} failed: HTTP {resp.status_code}")
|
||||
return resp.content
|
||||
|
||||
async def code_map(self, on: date) -> dict[str, str]:
|
||||
return parse_code_map(await self._get(DAILY_URL, {"date_req": _fmt(on)}))
|
||||
|
||||
async def quotes(self, cbr_id: str, start: date, end: date) -> list[CbrQuote]:
|
||||
collected: list[CbrQuote] = []
|
||||
for window_start, window_end in split_range(start, end):
|
||||
payload = await self._get(
|
||||
DYNAMIC_URL,
|
||||
{
|
||||
"date_req1": _fmt(window_start),
|
||||
"date_req2": _fmt(window_end),
|
||||
"VAL_NM_RQ": cbr_id,
|
||||
},
|
||||
)
|
||||
collected.extend(parse_dynamic(payload))
|
||||
return collected
|
||||
@@ -0,0 +1,161 @@
|
||||
"""The `cbr` source: official RUB rates for every currency the ledger actually uses.
|
||||
|
||||
What to fetch is derived from the data, not configured: the distinct currencies of
|
||||
`account.currency`, `cash_txn.income_currency` and `cash_txn.outcome_currency` (RUB itself
|
||||
is quoted as 1.0 by `pricing/fx.py`, so it is never requested).
|
||||
|
||||
Date range: from the earliest transaction minus 7 days (or today − 30 days when there are
|
||||
no transactions yet) up to today — `analytics.today_local()`, the deployment timezone, the
|
||||
same "today" the metrics end on. MSK runs ahead of UTC, so a UTC date would ask for one day
|
||||
less for the first hours of every Moscow morning. Incrementally, the cursor is the last date
|
||||
fetched and the next run starts 3 days earlier — CBR publishes the next business day's rate
|
||||
in the evening, and that overlap re-reads the tail cheaply. If the ledger grew *backwards* (an
|
||||
import of older history) the full range is used again, detected by comparing the wanted
|
||||
start with the earliest date already in `raw_cbr_rate`.
|
||||
|
||||
Currencies CBR does not quote (crypto, metals such as XAU) are reported in `warnings` and
|
||||
skipped — never an error, and never an invented rate.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, date, datetime, timedelta
|
||||
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.analytics import ledger_date_range, today_local
|
||||
from fintracker.models import Account, CashTxn, RawCbrRate
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.cbr.client import CbrClient, CbrQuote
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = "cbr"
|
||||
BASE = "RUB"
|
||||
LOOKBACK_DAYS = 7
|
||||
"""Margin before the first transaction, so a rate exists for its date even over a weekend."""
|
||||
CURSOR_OVERLAP_DAYS = 3
|
||||
BACKFILL_TOLERANCE_DAYS = 14
|
||||
"""How far the first stored quote may legitimately sit after the wanted start (holidays)."""
|
||||
NO_HISTORY_DAYS = 30
|
||||
CHUNK = 500
|
||||
|
||||
|
||||
class CbrSource:
|
||||
name = SOURCE
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
session = ctx.session
|
||||
today = today_local()
|
||||
currencies = await currencies_in_use(session)
|
||||
if not currencies:
|
||||
log.info("cbr: only %s in use, nothing to fetch", BASE)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts={"currencies": 0, "rates": 0},
|
||||
changed=False,
|
||||
)
|
||||
|
||||
start = await _start_date(session, ctx.cursor_before, today)
|
||||
warnings: list[str] = []
|
||||
stored = 0
|
||||
async with CbrClient() as client:
|
||||
codes = await client.code_map(today)
|
||||
fetched_currencies = 0
|
||||
for ccy in currencies:
|
||||
cbr_id = codes.get(ccy)
|
||||
if not cbr_id:
|
||||
warnings.append(f"{ccy} is not quoted by CBR — skipped (crypto or metal?)")
|
||||
continue
|
||||
quotes = await client.quotes(cbr_id, start, today)
|
||||
stored += await _store(session, ccy, quotes)
|
||||
fetched_currencies += 1
|
||||
|
||||
await session.commit()
|
||||
log.info(
|
||||
"cbr: %s rates for %s currencies, %s..%s%s",
|
||||
stored,
|
||||
fetched_currencies,
|
||||
start,
|
||||
today,
|
||||
f", skipped {len(warnings)}" if warnings else "",
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=today.isoformat(),
|
||||
counts={"currencies": fetched_currencies, "rates": stored},
|
||||
warnings=warnings,
|
||||
changed=stored > 0,
|
||||
)
|
||||
|
||||
|
||||
async def currencies_in_use(session: AsyncSession) -> list[str]:
|
||||
found: set[str] = set()
|
||||
rows = await session.execute(select(Account.currency).where(Account.currency.is_not(None)))
|
||||
found.update(code for code in rows.scalars().all() if code)
|
||||
# deleted transactions are tombstones no metric reads — their currency is not "in use"
|
||||
for column in (CashTxn.income_currency, CashTxn.outcome_currency):
|
||||
rows = await session.execute(
|
||||
select(column).where(column.is_not(None), CashTxn.deleted.is_(False)).distinct()
|
||||
)
|
||||
found.update(code for code in rows.scalars().all() if code)
|
||||
return sorted(code for code in found if code != BASE)
|
||||
|
||||
|
||||
async def _start_date(session: AsyncSession, cursor: str | None, today: date) -> date:
|
||||
first_txn, _ = await ledger_date_range(session)
|
||||
full_start = (
|
||||
first_txn - timedelta(days=LOOKBACK_DAYS)
|
||||
if first_txn is not None
|
||||
else today - timedelta(days=NO_HISTORY_DAYS)
|
||||
)
|
||||
cursor_date = _parse_date(cursor)
|
||||
if cursor_date is None:
|
||||
return full_start
|
||||
have_from = (await session.execute(select(func.min(RawCbrRate.rate_date)))).scalar_one_or_none()
|
||||
if have_from is None or full_start + timedelta(days=BACKFILL_TOLERANCE_DAYS) < have_from:
|
||||
# the ledger grew backwards (older history imported) — refetch from the new beginning.
|
||||
# The tolerance absorbs the normal case where the wanted start simply falls on a
|
||||
# weekend or a holiday stretch that CBR never quoted.
|
||||
return full_start
|
||||
return min(today, cursor_date - timedelta(days=CURSOR_OVERLAP_DAYS))
|
||||
|
||||
|
||||
def _parse_date(value: str | None) -> date | None:
|
||||
if not value:
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value)
|
||||
except ValueError:
|
||||
log.warning("cbr: unusable cursor %r, refetching the full range", value)
|
||||
return None
|
||||
|
||||
|
||||
async def _store(session: AsyncSession, ccy: str, quotes: list[CbrQuote]) -> int:
|
||||
rows = [
|
||||
{
|
||||
"rate_date": quote.rate_date,
|
||||
"ccy": ccy,
|
||||
"nominal": quote.nominal,
|
||||
"value": quote.value,
|
||||
"fetched_at": datetime.now(UTC),
|
||||
}
|
||||
for quote in quotes
|
||||
]
|
||||
if not rows:
|
||||
return 0
|
||||
for start in range(0, len(rows), CHUNK):
|
||||
chunk = rows[start : start + CHUNK]
|
||||
stmt = pg_insert(RawCbrRate).values(chunk)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["rate_date", "ccy"],
|
||||
set_={
|
||||
"nominal": stmt.excluded.nominal,
|
||||
"value": stmt.excluded.value,
|
||||
"fetched_at": stmt.excluded.fetched_at,
|
||||
},
|
||||
)
|
||||
await session.execute(stmt)
|
||||
return len(rows)
|
||||
@@ -0,0 +1,24 @@
|
||||
"""Registry of available sources. Later phases register zenmoney, cbr, tinvest, moex here."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fintracker.sources.base import Source
|
||||
|
||||
_SOURCES: dict[str, Source] = {}
|
||||
|
||||
|
||||
def register(source: Source) -> Source:
|
||||
_SOURCES[source.name] = source
|
||||
return source
|
||||
|
||||
|
||||
def unregister(name: str) -> None:
|
||||
_SOURCES.pop(name, None)
|
||||
|
||||
|
||||
def get(name: str) -> Source:
|
||||
return _SOURCES[name]
|
||||
|
||||
|
||||
def names() -> list[str]:
|
||||
return sorted(_SOURCES)
|
||||
@@ -0,0 +1,8 @@
|
||||
"""ZenMoney source package. Importing it registers the source."""
|
||||
|
||||
from fintracker.sources.registry import register
|
||||
from fintracker.sources.zenmoney.sync import ZenmoneySource
|
||||
|
||||
register(ZenmoneySource())
|
||||
|
||||
__all__ = ["ZenmoneySource"]
|
||||
@@ -0,0 +1,280 @@
|
||||
"""HTTP access to the ZenMoney API (plan § "ZenMoney").
|
||||
|
||||
Reading is possible only through one endpoint: `POST https://api.zenmoney.ru/v8/diff/`.
|
||||
It is incremental by `serverTimestamp`; the first call (cursor 0) additionally asks for
|
||||
`forceFetch` of every entity type. The answer holds the same entity arrays (only rows
|
||||
changed since the cursor), a `deletion` list and the new `serverTimestamp`.
|
||||
|
||||
NETWORK NOTE: this host exports http_proxy/https_proxy/all_proxy and api.zenmoney.ru IS
|
||||
reachable through that proxy, so the client keeps httpx's default `trust_env=True`.
|
||||
The CBR client does the opposite — cbr.ru fails the TLS handshake through the proxy,
|
||||
see `sources/cbr/client.py`.
|
||||
|
||||
AUTH. Access tokens live 86400 s. Two modes, picked from settings:
|
||||
|
||||
* static token — `ZENMONEY_TOKEN` (e.g. minted at zerro.app/token). There is nothing to
|
||||
rotate, so HTTP 401 can only be reported as "renew ZENMONEY_TOKEN".
|
||||
* OAuth rotation — enabled when `ZENMONEY_CLIENT_ID` and `ZENMONEY_CLIENT_SECRET` are set.
|
||||
The `{access_token, refresh_token, expires_at}` triple lives in `source_credential`
|
||||
(source='zenmoney'), seeded from `ZENMONEY_REFRESH_TOKEN`. Before a sync an expired
|
||||
(or missing) access token is refreshed with
|
||||
`POST https://api.zenmoney.ru/oauth2/token/` form-encoded
|
||||
`grant_type=refresh_token&refresh_token=…&client_id=…&client_secret=…`; the response
|
||||
`{access_token, refresh_token, expires_in, token_type}` replaces the stored pair.
|
||||
A 401 on the diff call triggers exactly one extra refresh + retry.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
from dataclasses import dataclass
|
||||
from datetime import UTC, datetime, timedelta
|
||||
from types import TracebackType
|
||||
from typing import Any, Self
|
||||
|
||||
import httpx
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.config import Settings
|
||||
from fintracker.models import SourceCredential
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
SOURCE = "zenmoney"
|
||||
DIFF_URL = "https://api.zenmoney.ru/v8/diff/"
|
||||
TOKEN_URL = "https://api.zenmoney.ru/oauth2/token/"
|
||||
TIMEOUT = 120.0
|
||||
|
||||
ENTITY_TYPES: tuple[str, ...] = (
|
||||
"instrument",
|
||||
"company",
|
||||
"user",
|
||||
"account",
|
||||
"tag",
|
||||
"merchant",
|
||||
"budget",
|
||||
"reminder",
|
||||
"reminderMarker",
|
||||
"transaction",
|
||||
)
|
||||
"""Every entity type /v8/diff/ can return; also the `forceFetch` list on a full pull."""
|
||||
|
||||
EXPIRY_SKEW = timedelta(minutes=5)
|
||||
DEFAULT_TOKEN_TTL = 86400
|
||||
|
||||
|
||||
class ZenmoneyError(RuntimeError):
|
||||
"""Upstream refused or misbehaved."""
|
||||
|
||||
|
||||
class ZenmoneyAuthError(ZenmoneyError):
|
||||
"""Credentials are missing, expired or rejected."""
|
||||
|
||||
|
||||
@dataclass
|
||||
class TokenBundle:
|
||||
access_token: str
|
||||
refresh_token: str | None = None
|
||||
expires_at: datetime | None = None
|
||||
|
||||
@property
|
||||
def expired(self) -> bool:
|
||||
if self.expires_at is None:
|
||||
return False
|
||||
return self.expires_at - EXPIRY_SKEW <= datetime.now(UTC)
|
||||
|
||||
def to_payload(self) -> dict[str, Any]:
|
||||
return {
|
||||
"access_token": self.access_token,
|
||||
"refresh_token": self.refresh_token,
|
||||
"expires_at": self.expires_at.isoformat() if self.expires_at else None,
|
||||
}
|
||||
|
||||
@classmethod
|
||||
def from_payload(cls, payload: dict[str, Any] | None) -> TokenBundle | None:
|
||||
if not payload or not payload.get("access_token"):
|
||||
return None
|
||||
raw_expires = payload.get("expires_at")
|
||||
expires_at: datetime | None = None
|
||||
if isinstance(raw_expires, str):
|
||||
try:
|
||||
expires_at = datetime.fromisoformat(raw_expires)
|
||||
except ValueError:
|
||||
expires_at = None
|
||||
if expires_at is not None and expires_at.tzinfo is None:
|
||||
expires_at = expires_at.replace(tzinfo=UTC)
|
||||
return cls(
|
||||
access_token=str(payload["access_token"]),
|
||||
refresh_token=payload.get("refresh_token"),
|
||||
expires_at=expires_at,
|
||||
)
|
||||
|
||||
|
||||
class ZenmoneyClient:
|
||||
"""One diff call per sync, plus the token rotation around it.
|
||||
|
||||
Needs the sync session because in OAuth mode the rotated pair must be persisted in
|
||||
`source_credential` — the worker runs unattended, so a token it fetched and lost
|
||||
would strand the next run.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
settings: Settings,
|
||||
session: AsyncSession,
|
||||
*,
|
||||
http: httpx.AsyncClient | None = None,
|
||||
) -> None:
|
||||
self._settings = settings
|
||||
self._session = session
|
||||
self._http = http
|
||||
self._owns_http = http is None
|
||||
self._token: TokenBundle | None = None
|
||||
|
||||
# --- lifecycle ---------------------------------------------------------------
|
||||
|
||||
@property
|
||||
def oauth_mode(self) -> bool:
|
||||
return bool(self._settings.zenmoney_client_id and self._settings.zenmoney_client_secret)
|
||||
|
||||
def _client(self) -> httpx.AsyncClient:
|
||||
if self._http is None:
|
||||
# trust_env stays on: ZenMoney is reachable only through the host proxy
|
||||
self._http = httpx.AsyncClient(timeout=TIMEOUT)
|
||||
return self._http
|
||||
|
||||
async def aclose(self) -> None:
|
||||
if self._http is not None and self._owns_http:
|
||||
await self._http.aclose()
|
||||
self._http = None
|
||||
|
||||
async def __aenter__(self) -> Self:
|
||||
return self
|
||||
|
||||
async def __aexit__(
|
||||
self,
|
||||
exc_type: type[BaseException] | None,
|
||||
exc: BaseException | None,
|
||||
tb: TracebackType | None,
|
||||
) -> None:
|
||||
await self.aclose()
|
||||
|
||||
# --- diff --------------------------------------------------------------------
|
||||
|
||||
async def diff(self, server_timestamp: int) -> dict[str, Any]:
|
||||
body: dict[str, Any] = {
|
||||
"currentClientTimestamp": int(datetime.now(UTC).timestamp()),
|
||||
"serverTimestamp": server_timestamp,
|
||||
}
|
||||
if server_timestamp == 0:
|
||||
body["forceFetch"] = list(ENTITY_TYPES)
|
||||
|
||||
token = await self.access_token()
|
||||
resp = await self._post_diff(token, body)
|
||||
if resp.status_code == 401:
|
||||
if not self.oauth_mode:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZenMoney rejected the static access token (HTTP 401). ZenMoney access "
|
||||
"tokens live 24 hours — renew ZENMONEY_TOKEN (e.g. at zerro.app/token) "
|
||||
"and put the new value in .env, or configure ZENMONEY_CLIENT_ID / "
|
||||
"ZENMONEY_CLIENT_SECRET / ZENMONEY_REFRESH_TOKEN for automatic rotation."
|
||||
)
|
||||
log.info("zenmoney: diff got 401, refreshing the access token once")
|
||||
refreshed = await self._refresh_token()
|
||||
resp = await self._post_diff(refreshed.access_token, body)
|
||||
if resp.status_code == 401:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZenMoney rejected a freshly refreshed access token (HTTP 401); check "
|
||||
"ZENMONEY_CLIENT_ID / ZENMONEY_CLIENT_SECRET and the refresh token "
|
||||
"stored in source_credential."
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise ZenmoneyError(
|
||||
f"ZenMoney /v8/diff/ failed: HTTP {resp.status_code} {resp.text[:500]}"
|
||||
)
|
||||
data = json.loads(resp.text)
|
||||
if not isinstance(data, dict):
|
||||
raise ZenmoneyError("ZenMoney /v8/diff/ returned a non-object body")
|
||||
return data
|
||||
|
||||
async def _post_diff(self, token: str, body: dict[str, Any]) -> httpx.Response:
|
||||
return await self._client().post(
|
||||
DIFF_URL,
|
||||
headers={"Authorization": f"Bearer {token}"},
|
||||
json=body,
|
||||
)
|
||||
|
||||
# --- tokens ------------------------------------------------------------------
|
||||
|
||||
async def access_token(self) -> str:
|
||||
if not self.oauth_mode:
|
||||
if not self._settings.zenmoney_token:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZENMONEY_TOKEN is not set — put a ZenMoney access token in .env "
|
||||
"(e.g. from zerro.app/token), or configure ZENMONEY_CLIENT_ID / "
|
||||
"ZENMONEY_CLIENT_SECRET / ZENMONEY_REFRESH_TOKEN for OAuth rotation."
|
||||
)
|
||||
return self._settings.zenmoney_token
|
||||
|
||||
bundle = self._token or await self._load_credential()
|
||||
if bundle is None or bundle.expired:
|
||||
bundle = await self._refresh_token(bundle)
|
||||
self._token = bundle
|
||||
return bundle.access_token
|
||||
|
||||
async def _load_credential(self) -> TokenBundle | None:
|
||||
row = await self._session.get(SourceCredential, SOURCE)
|
||||
return TokenBundle.from_payload(row.payload if row else None)
|
||||
|
||||
async def _refresh_token(self, bundle: TokenBundle | None = None) -> TokenBundle:
|
||||
settings = self._settings
|
||||
known = bundle or self._token or await self._load_credential()
|
||||
refresh_token = (known.refresh_token if known else None) or settings.zenmoney_refresh_token
|
||||
if not refresh_token:
|
||||
raise ZenmoneyAuthError(
|
||||
"OAuth mode is configured but no refresh token is available — set "
|
||||
"ZENMONEY_REFRESH_TOKEN in .env to seed source_credential."
|
||||
)
|
||||
resp = await self._client().post(
|
||||
TOKEN_URL,
|
||||
data={
|
||||
"grant_type": "refresh_token",
|
||||
"refresh_token": refresh_token,
|
||||
"client_id": settings.zenmoney_client_id or "",
|
||||
"client_secret": settings.zenmoney_client_secret or "",
|
||||
},
|
||||
)
|
||||
if resp.status_code >= 400:
|
||||
raise ZenmoneyAuthError(
|
||||
"ZenMoney refused to refresh the access token: "
|
||||
f"HTTP {resp.status_code} {resp.text[:500]}"
|
||||
)
|
||||
data = resp.json()
|
||||
access_token = data.get("access_token")
|
||||
if not access_token:
|
||||
raise ZenmoneyAuthError("ZenMoney token response carried no access_token")
|
||||
try:
|
||||
ttl = int(data.get("expires_in") or DEFAULT_TOKEN_TTL)
|
||||
except (TypeError, ValueError):
|
||||
ttl = DEFAULT_TOKEN_TTL
|
||||
fresh = TokenBundle(
|
||||
access_token=str(access_token),
|
||||
refresh_token=data.get("refresh_token") or refresh_token,
|
||||
expires_at=datetime.now(UTC) + timedelta(seconds=ttl),
|
||||
)
|
||||
await self._store_credential(fresh)
|
||||
self._token = fresh
|
||||
return fresh
|
||||
|
||||
async def _store_credential(self, bundle: TokenBundle) -> None:
|
||||
payload = bundle.to_payload()
|
||||
stmt = pg_insert(SourceCredential).values(source=SOURCE, payload=payload)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["source"],
|
||||
set_={"payload": stmt.excluded.payload, "updated_at": datetime.now(UTC)},
|
||||
)
|
||||
await self._session.execute(stmt)
|
||||
# committed right away: a token we fetched but lost would strand the next run
|
||||
await self._session.commit()
|
||||
@@ -0,0 +1,483 @@
|
||||
"""Raw ZenMoney entities -> core tables (plan §1.2, §1.7).
|
||||
|
||||
The mapper never looks at the diff itself: it rebuilds the core rows from the full
|
||||
`raw_zenmoney_entity` table, so it is idempotent and self-healing (a mapping bug is fixed
|
||||
by re-running it, no re-download). Personal volumes are ~10^4 transactions, so a full
|
||||
upsert per sync is cheap; `sync.py` skips the mapper entirely when the diff was empty.
|
||||
|
||||
Rows deleted upstream are gone from the raw table, so they are never re-created here —
|
||||
the flags `sync.py` sets while processing `deletion` survive.
|
||||
|
||||
Money: every amount goes through `_money()` into `Decimal`. ZenMoney sends amounts as JSON
|
||||
numbers, so they arrive from JSONB as Python floats; `Decimal(str(value))` round-trips the
|
||||
printed (≤2 decimals) value exactly. Nothing float-valued is ever stored.
|
||||
|
||||
What is deliberately NOT touched here (owned by `analytics/classify.py`): `payee_canonical`,
|
||||
`is_one_off`, `trip_id`, and the refined `flow_type`/`category_id`. The mapper only lays
|
||||
down the base values ZenMoney itself implies.
|
||||
|
||||
Account fields the USER owns (`PATCH /accounts/{id}`): `name`, `role`, `include_in_net_worth`
|
||||
— plus `mirror_of_account_id` and `primary_event_source`, which the mapper never produces at
|
||||
all. They are seeded on INSERT and never overwritten on conflict, so a sync cannot undo an
|
||||
edit. The single exception: an account the source marks `archive` is forced out of net worth,
|
||||
the same way `sync.py` treats a deletion.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
from datetime import UTC, date, datetime
|
||||
from decimal import Decimal, InvalidOperation
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import case, delete, false, select, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
from sqlalchemy.ext.asyncio import AsyncSession
|
||||
|
||||
from fintracker.config import Settings
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
AccountKind,
|
||||
AccountRole,
|
||||
CashTxn,
|
||||
CashTxnTag,
|
||||
Category,
|
||||
FlowType,
|
||||
Merchant,
|
||||
RawZenmoneyEntity,
|
||||
)
|
||||
|
||||
SOURCE = "zenmoney"
|
||||
CHUNK = 500
|
||||
|
||||
KIND_BY_TYPE: dict[str, AccountKind] = {
|
||||
"cash": AccountKind.zm_cash,
|
||||
"ccard": AccountKind.zm_card,
|
||||
"checking": AccountKind.zm_checking,
|
||||
"deposit": AccountKind.zm_deposit,
|
||||
"loan": AccountKind.zm_loan,
|
||||
"emoney": AccountKind.zm_emoney,
|
||||
"debt": AccountKind.zm_debt,
|
||||
}
|
||||
"""ZenMoney `account.type` -> `AccountKind`. Only `ccard` -> `zm_card` is not a literal
|
||||
"zm_" + type: the enum in `models/accounts.py` spells the card kind without the extra c."""
|
||||
|
||||
DEBT_TYPES = frozenset({"loan", "debt"})
|
||||
DEPOSIT_FIELDS = (
|
||||
"capitalization",
|
||||
"percent",
|
||||
"startDate",
|
||||
"endDateOffset",
|
||||
"endDateOffsetInterval",
|
||||
"payoffStep",
|
||||
"payoffInterval",
|
||||
)
|
||||
META_FIELDS = ("reminderMarker", "latitude", "longitude")
|
||||
|
||||
|
||||
def _money(value: Any) -> Decimal | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
if isinstance(value, Decimal):
|
||||
return value
|
||||
try:
|
||||
return Decimal(str(value))
|
||||
except (InvalidOperation, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _int(value: Any) -> int | None:
|
||||
if value is None or isinstance(value, bool):
|
||||
return None
|
||||
try:
|
||||
return int(value)
|
||||
except (TypeError, ValueError):
|
||||
return None
|
||||
|
||||
|
||||
def _ccy(codes: dict[int, str], value: Any) -> str | None:
|
||||
"""Instrument id -> currency code. `None` for an absent or unknown instrument; id 0 is a
|
||||
perfectly valid key and must not collapse into a sentinel."""
|
||||
ident = _int(value)
|
||||
return codes.get(ident) if ident is not None else None
|
||||
|
||||
|
||||
def _day(value: Any) -> date | None:
|
||||
if not isinstance(value, str):
|
||||
return None
|
||||
try:
|
||||
return date.fromisoformat(value[:10])
|
||||
except ValueError:
|
||||
return None
|
||||
|
||||
|
||||
def _chunks(
|
||||
rows: Sequence[dict[str, Any]], size: int = CHUNK
|
||||
) -> Iterator[Sequence[dict[str, Any]]]:
|
||||
for start in range(0, len(rows), size):
|
||||
yield rows[start : start + size]
|
||||
|
||||
|
||||
async def _upsert(
|
||||
session: AsyncSession,
|
||||
model: type[Any],
|
||||
rows: Sequence[dict[str, Any]],
|
||||
conflict: list[str],
|
||||
update_cols: Iterable[str],
|
||||
set_extra: Callable[[Any], dict[str, Any]] | None = None,
|
||||
) -> int:
|
||||
"""Postgres INSERT … ON CONFLICT DO UPDATE, chunked. `update_cols` deliberately omits
|
||||
the columns other layers own, so re-mapping cannot clobber them. `set_extra` gets the
|
||||
insert statement and returns SET entries that are not a plain copy from `excluded`."""
|
||||
cols = list(update_cols)
|
||||
for chunk in _chunks(rows):
|
||||
stmt = pg_insert(model).values(list(chunk))
|
||||
set_: dict[str, Any] = {name: getattr(stmt.excluded, name) for name in cols}
|
||||
if set_extra is not None:
|
||||
set_.update(set_extra(stmt))
|
||||
stmt = stmt.on_conflict_do_update(index_elements=conflict, set_=set_)
|
||||
await session.execute(stmt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def _source_ids(session: AsyncSession, model: type[Any]) -> dict[str, int]:
|
||||
rows = await session.execute(select(model.source_id, model.id).where(model.source == SOURCE))
|
||||
return {source_id: row_id for source_id, row_id in rows.all()}
|
||||
|
||||
|
||||
async def load_raw(session: AsyncSession) -> dict[str, list[dict[str, Any]]]:
|
||||
"""Every raw entity, grouped by type."""
|
||||
rows = await session.execute(select(RawZenmoneyEntity.entity_type, RawZenmoneyEntity.payload))
|
||||
grouped: dict[str, list[dict[str, Any]]] = {}
|
||||
for entity_type, payload in rows.all():
|
||||
if isinstance(payload, dict):
|
||||
grouped.setdefault(entity_type, []).append(payload)
|
||||
return grouped
|
||||
|
||||
|
||||
def instrument_codes(instruments: list[dict[str, Any]]) -> dict[int, str]:
|
||||
"""ZenMoney instrument id -> 3-letter code ('RUB')."""
|
||||
codes: dict[int, str] = {}
|
||||
for payload in instruments:
|
||||
ident = _int(payload.get("id"))
|
||||
code = payload.get("shortTitle") or payload.get("title")
|
||||
if ident is not None and isinstance(code, str) and code:
|
||||
codes[ident] = code.upper()[:3]
|
||||
return codes
|
||||
|
||||
|
||||
async def map_all(session: AsyncSession, settings: Settings) -> tuple[dict[str, int], list[str]]:
|
||||
warnings: list[str] = []
|
||||
raw = await load_raw(session)
|
||||
codes = instrument_codes(raw.get("instrument", []))
|
||||
|
||||
categories = await _map_categories(session, raw.get("tag", []))
|
||||
merchants = await _map_merchants(session, raw.get("merchant", []))
|
||||
accounts = await _map_accounts(session, raw.get("account", []), codes, settings, warnings)
|
||||
transactions = await _map_transactions(
|
||||
session,
|
||||
raw.get("transaction", []),
|
||||
codes=codes,
|
||||
accounts=accounts,
|
||||
categories=categories,
|
||||
merchants=merchants,
|
||||
warnings=warnings,
|
||||
)
|
||||
counts = {
|
||||
"instruments": len(codes),
|
||||
"categories": len(categories),
|
||||
"merchants": len(merchants),
|
||||
"accounts": len(accounts),
|
||||
"transactions": transactions,
|
||||
}
|
||||
return counts, warnings
|
||||
|
||||
|
||||
# --- tags -> category -----------------------------------------------------------------
|
||||
|
||||
|
||||
async def _map_categories(session: AsyncSession, tags: list[dict[str, Any]]) -> dict[str, int]:
|
||||
"""Two passes: upsert the rows, then resolve `parent_id` (ZenMoney nests one level)."""
|
||||
rows = [
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": str(payload["id"]),
|
||||
"name": str(payload.get("title") or ""),
|
||||
"icon": payload.get("icon"),
|
||||
"color": _int(payload.get("color")),
|
||||
"show_income": bool(payload.get("showIncome", True)),
|
||||
"show_outcome": bool(payload.get("showOutcome", True)),
|
||||
"archived": False,
|
||||
}
|
||||
for payload in tags
|
||||
if payload.get("id") is not None
|
||||
]
|
||||
if rows:
|
||||
await _upsert(
|
||||
session,
|
||||
Category,
|
||||
rows,
|
||||
["source", "source_id"],
|
||||
("name", "icon", "color", "show_income", "show_outcome"),
|
||||
)
|
||||
ids = await _source_ids(session, Category)
|
||||
if not ids:
|
||||
return ids
|
||||
|
||||
parents = [
|
||||
{"id": ids[str(payload["id"])], "parent_id": ids.get(str(payload.get("parent")))}
|
||||
for payload in tags
|
||||
if payload.get("id") is not None and str(payload["id"]) in ids
|
||||
]
|
||||
if parents:
|
||||
await session.execute(update(Category), parents)
|
||||
return ids
|
||||
|
||||
|
||||
async def _map_merchants(session: AsyncSession, merchants: list[dict[str, Any]]) -> dict[str, int]:
|
||||
rows = [
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": str(payload["id"]),
|
||||
"name": str(payload.get("title") or ""),
|
||||
}
|
||||
for payload in merchants
|
||||
if payload.get("id") is not None
|
||||
]
|
||||
if rows:
|
||||
await _upsert(session, Merchant, rows, ["source", "source_id"], ("name",))
|
||||
return await _source_ids(session, Merchant)
|
||||
|
||||
|
||||
# --- accounts -------------------------------------------------------------------------
|
||||
|
||||
|
||||
def account_role(acc_type: str, savings: bool) -> AccountRole:
|
||||
"""loan/debt are what we owe; an explicit savings flag or a deposit is savings money;
|
||||
everything else is spendable. `investment` is left to broker accounts (phase 2), and
|
||||
`mirror_of_account_id` / `primary_event_source` stay whatever the user set."""
|
||||
if acc_type in DEBT_TYPES:
|
||||
return AccountRole.debt
|
||||
if savings or acc_type == "deposit":
|
||||
return AccountRole.savings
|
||||
return AccountRole.liquid
|
||||
|
||||
|
||||
async def _map_accounts(
|
||||
session: AsyncSession,
|
||||
accounts: list[dict[str, Any]],
|
||||
codes: dict[int, str],
|
||||
settings: Settings,
|
||||
warnings: list[str],
|
||||
) -> dict[str, int]:
|
||||
observed_at = datetime.now(UTC)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for payload in accounts:
|
||||
if payload.get("id") is None:
|
||||
continue
|
||||
source_id = str(payload["id"])
|
||||
acc_type = str(payload.get("type") or "cash")
|
||||
kind = KIND_BY_TYPE.get(acc_type)
|
||||
if kind is None:
|
||||
warnings.append(
|
||||
f"account {source_id}: unknown ZenMoney type {acc_type!r}, stored as zm_cash"
|
||||
)
|
||||
kind = AccountKind.zm_cash
|
||||
currency = _ccy(codes, payload.get("instrument"))
|
||||
if not currency:
|
||||
currency = settings.base_currency
|
||||
warnings.append(
|
||||
f"account {source_id}: instrument {payload.get('instrument')!r} is unknown, "
|
||||
f"currency assumed {currency}"
|
||||
)
|
||||
archived = bool(payload.get("archive"))
|
||||
terms = {
|
||||
field: payload[field] for field in DEPOSIT_FIELDS if payload.get(field) is not None
|
||||
}
|
||||
rows.append(
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": source_id,
|
||||
"kind": kind,
|
||||
# name / role / include_in_net_worth are seeded here and then owned by the
|
||||
# user: they are absent from the ON CONFLICT set below
|
||||
"name": str(payload.get("title") or source_id),
|
||||
"currency": currency,
|
||||
"role": account_role(acc_type, bool(payload.get("savings"))),
|
||||
# inBalance is ZenMoney's own "counts towards my balance" switch
|
||||
"include_in_net_worth": bool(payload.get("inBalance", True)) and not archived,
|
||||
"archived": archived,
|
||||
"opened_at": _day(payload.get("startDate")),
|
||||
"deposit_terms": terms or None,
|
||||
"balance": _money(payload.get("balance")),
|
||||
"start_balance": _money(payload.get("startBalance")),
|
||||
"credit_limit": _money(payload.get("creditLimit")),
|
||||
"balance_as_of": observed_at,
|
||||
}
|
||||
)
|
||||
if rows:
|
||||
await _upsert(
|
||||
session,
|
||||
Account,
|
||||
rows,
|
||||
["source", "source_id"],
|
||||
(
|
||||
"kind",
|
||||
"currency",
|
||||
"archived",
|
||||
"opened_at",
|
||||
"deposit_terms",
|
||||
"balance",
|
||||
"start_balance",
|
||||
"credit_limit",
|
||||
"balance_as_of",
|
||||
),
|
||||
# archived upstream still forces the account out of net worth (same as a
|
||||
# deletion); otherwise the user's own switch wins
|
||||
set_extra=lambda stmt: {
|
||||
"include_in_net_worth": case(
|
||||
(stmt.excluded.archived, false()),
|
||||
else_=Account.include_in_net_worth,
|
||||
)
|
||||
},
|
||||
)
|
||||
return await _source_ids(session, Account)
|
||||
|
||||
|
||||
# --- transactions ---------------------------------------------------------------------
|
||||
|
||||
|
||||
def flow_type(income: Decimal, outcome: Decimal, deleted: bool) -> FlowType:
|
||||
"""The base classification ZenMoney itself implies. Rules (savings_transfer, one_off,
|
||||
broker flows) refine it later in `analytics/classify.py`."""
|
||||
if deleted:
|
||||
return FlowType.deleted
|
||||
if income > 0 and outcome > 0:
|
||||
return FlowType.internal_transfer
|
||||
if outcome > 0:
|
||||
return FlowType.expense
|
||||
if income > 0:
|
||||
return FlowType.income
|
||||
return FlowType.other
|
||||
|
||||
|
||||
async def _map_transactions(
|
||||
session: AsyncSession,
|
||||
transactions: list[dict[str, Any]],
|
||||
*,
|
||||
codes: dict[int, str],
|
||||
accounts: dict[str, int],
|
||||
categories: dict[str, int],
|
||||
merchants: dict[str, int],
|
||||
warnings: list[str],
|
||||
) -> int:
|
||||
rows: list[dict[str, Any]] = []
|
||||
tags_by_source_id: dict[str, list[int]] = {}
|
||||
for payload in transactions:
|
||||
if payload.get("id") is None:
|
||||
continue
|
||||
source_id = str(payload["id"])
|
||||
day = _day(payload.get("date"))
|
||||
if day is None:
|
||||
warnings.append(f"transaction {source_id}: unparsable date {payload.get('date')!r}")
|
||||
continue
|
||||
created = _int(payload.get("created")) or _int(payload.get("changed"))
|
||||
ts = (
|
||||
datetime.fromtimestamp(created, UTC)
|
||||
if created is not None
|
||||
else datetime.combine(day, datetime.min.time(), UTC)
|
||||
)
|
||||
income = _money(payload.get("income")) or Decimal(0)
|
||||
outcome = _money(payload.get("outcome")) or Decimal(0)
|
||||
deleted = bool(payload.get("deleted"))
|
||||
|
||||
tag_ids = [
|
||||
categories[str(tag)] for tag in (payload.get("tag") or []) if str(tag) in categories
|
||||
]
|
||||
tags_by_source_id[source_id] = tag_ids
|
||||
primary_category_id = tag_ids[0] if tag_ids else None
|
||||
meta = {field: payload[field] for field in META_FIELDS if payload.get(field) is not None}
|
||||
|
||||
rows.append(
|
||||
{
|
||||
"source": SOURCE,
|
||||
"source_id": source_id,
|
||||
"ts": ts,
|
||||
"date": day,
|
||||
"income": income,
|
||||
"income_currency": _ccy(codes, payload.get("incomeInstrument")),
|
||||
"income_account_id": accounts.get(str(payload.get("incomeAccount"))),
|
||||
"outcome": outcome,
|
||||
"outcome_currency": _ccy(codes, payload.get("outcomeInstrument")),
|
||||
"outcome_account_id": accounts.get(str(payload.get("outcomeAccount"))),
|
||||
"op_income": _money(payload.get("opIncome")),
|
||||
"op_income_currency": _ccy(codes, payload.get("opIncomeInstrument")),
|
||||
"op_outcome": _money(payload.get("opOutcome")),
|
||||
"op_outcome_currency": _ccy(codes, payload.get("opOutcomeInstrument")),
|
||||
"payee": payload.get("payee"),
|
||||
"original_payee": payload.get("originalPayee"),
|
||||
"merchant_id": merchants.get(str(payload.get("merchant"))),
|
||||
"comment": payload.get("comment"),
|
||||
"mcc": _int(payload.get("mcc")),
|
||||
"hold": bool(payload.get("hold")),
|
||||
"deleted": deleted,
|
||||
"changed": _int(payload.get("changed")),
|
||||
"primary_category_id": primary_category_id,
|
||||
"flow_type": flow_type(income, outcome, deleted),
|
||||
"category_id": primary_category_id,
|
||||
"is_one_off": False,
|
||||
}
|
||||
| {"meta": meta or None}
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
|
||||
await _upsert(
|
||||
session,
|
||||
CashTxn,
|
||||
rows,
|
||||
["source", "source_id"],
|
||||
(
|
||||
"ts",
|
||||
"date",
|
||||
"income",
|
||||
"income_currency",
|
||||
"income_account_id",
|
||||
"outcome",
|
||||
"outcome_currency",
|
||||
"outcome_account_id",
|
||||
"op_income",
|
||||
"op_income_currency",
|
||||
"op_outcome",
|
||||
"op_outcome_currency",
|
||||
"payee",
|
||||
"original_payee",
|
||||
"merchant_id",
|
||||
"comment",
|
||||
"mcc",
|
||||
"hold",
|
||||
"deleted",
|
||||
"changed",
|
||||
"primary_category_id",
|
||||
"flow_type",
|
||||
"category_id",
|
||||
"meta",
|
||||
),
|
||||
)
|
||||
|
||||
txn_ids = await _source_ids(session, CashTxn)
|
||||
links: list[dict[str, Any]] = []
|
||||
for source_id, tag_ids in tags_by_source_id.items():
|
||||
txn_id = txn_ids.get(source_id)
|
||||
if txn_id is None:
|
||||
continue
|
||||
links.extend(
|
||||
{"txn_id": txn_id, "ord": ord_, "category_id": category_id}
|
||||
for ord_, category_id in enumerate(tag_ids)
|
||||
)
|
||||
mapped_ids = [txn_ids[s] for s in tags_by_source_id if s in txn_ids]
|
||||
if mapped_ids:
|
||||
await session.execute(delete(CashTxnTag).where(CashTxnTag.txn_id.in_(mapped_ids)))
|
||||
for chunk in _chunks(links):
|
||||
await session.execute(pg_insert(CashTxnTag).values(list(chunk)))
|
||||
return len(rows)
|
||||
@@ -0,0 +1,169 @@
|
||||
"""The `zenmoney` source: one /v8/diff/ call per run, raw tier, then the core mapping.
|
||||
|
||||
Algorithm (plan §1.7):
|
||||
|
||||
1. POST /v8/diff/ with the stored cursor (`serverTimestamp`; `forceFetch` when it is 0).
|
||||
2. Upsert every returned entity into `raw_zenmoney_entity` by (entity_type, id).
|
||||
3. Apply `deletion`: drop the raw row, remember the fact in `raw_zenmoney_deletion`, and
|
||||
flag the core row — a deleted transaction becomes `deleted` (flow_type `deleted`), a
|
||||
deleted account or tag becomes archived.
|
||||
4. Re-map the raw tables into the core tables (see `mapper.py`) — always from the full raw
|
||||
tier, never from the diff alone, so the result does not depend on how the history was
|
||||
downloaded.
|
||||
5. Store the new `serverTimestamp` as the cursor.
|
||||
|
||||
Re-runs are cheap and inert: an empty diff (no entities, no deletions) skips the mapper
|
||||
altogether and reports `changed=False`, so the worker also skips the metrics refresh.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from datetime import UTC, datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import delete, update
|
||||
from sqlalchemy.dialects.postgresql import insert as pg_insert
|
||||
|
||||
from fintracker.models import (
|
||||
Account,
|
||||
CashTxn,
|
||||
Category,
|
||||
FlowType,
|
||||
RawZenmoneyDeletion,
|
||||
RawZenmoneyEntity,
|
||||
)
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.sources.zenmoney import mapper
|
||||
from fintracker.sources.zenmoney.client import ENTITY_TYPES, SOURCE, ZenmoneyClient
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ZenmoneySource:
|
||||
name = SOURCE
|
||||
|
||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||
cursor_before = _as_timestamp(ctx.cursor_before)
|
||||
async with ZenmoneyClient(ctx.settings, ctx.session) as client:
|
||||
data = await client.diff(cursor_before)
|
||||
|
||||
raw_upserted = await _upsert_entities(ctx, data)
|
||||
deleted = await _apply_deletions(ctx, data)
|
||||
changed = bool(raw_upserted or deleted)
|
||||
|
||||
counts: dict[str, int] = {"raw_upserted": raw_upserted, "deleted": deleted}
|
||||
warnings: list[str] = []
|
||||
if changed:
|
||||
mapped, warnings = await mapper.map_all(ctx.session, ctx.settings)
|
||||
counts.update(mapped)
|
||||
await ctx.session.commit()
|
||||
|
||||
cursor_after = data.get("serverTimestamp")
|
||||
log.info(
|
||||
"zenmoney: %s raw rows, %s deletions, cursor %s -> %s",
|
||||
raw_upserted,
|
||||
deleted,
|
||||
cursor_before,
|
||||
cursor_after,
|
||||
)
|
||||
return SyncResult(
|
||||
cursor_after=str(int(cursor_after)) if cursor_after is not None else None,
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=changed,
|
||||
)
|
||||
|
||||
|
||||
def _as_timestamp(cursor: str | None) -> int:
|
||||
if not cursor:
|
||||
return 0
|
||||
try:
|
||||
return int(cursor)
|
||||
except ValueError:
|
||||
log.warning("zenmoney: unusable cursor %r, doing a full pull", cursor)
|
||||
return 0
|
||||
|
||||
|
||||
async def _upsert_entities(ctx: SyncContext, data: dict[str, Any]) -> int:
|
||||
ingested_at = datetime.now(UTC)
|
||||
rows: list[dict[str, Any]] = []
|
||||
for entity_type in ENTITY_TYPES:
|
||||
for payload in data.get(entity_type) or []:
|
||||
if not isinstance(payload, dict) or payload.get("id") is None:
|
||||
continue
|
||||
rows.append(
|
||||
{
|
||||
"entity_type": entity_type,
|
||||
"id": str(payload["id"]),
|
||||
"changed": payload.get("changed"),
|
||||
"payload": payload,
|
||||
"ingested_at": ingested_at,
|
||||
}
|
||||
)
|
||||
if not rows:
|
||||
return 0
|
||||
for start in range(0, len(rows), mapper.CHUNK):
|
||||
chunk = rows[start : start + mapper.CHUNK]
|
||||
stmt = pg_insert(RawZenmoneyEntity).values(chunk)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["entity_type", "id"],
|
||||
set_={
|
||||
"changed": stmt.excluded.changed,
|
||||
"payload": stmt.excluded.payload,
|
||||
"ingested_at": stmt.excluded.ingested_at,
|
||||
},
|
||||
)
|
||||
await ctx.session.execute(stmt)
|
||||
return len(rows)
|
||||
|
||||
|
||||
async def _apply_deletions(ctx: SyncContext, data: dict[str, Any]) -> int:
|
||||
deletions = [d for d in (data.get("deletion") or []) if isinstance(d, dict)]
|
||||
if not deletions:
|
||||
return 0
|
||||
session = ctx.session
|
||||
count = 0
|
||||
for item in deletions:
|
||||
entity_type = str(item.get("object") or "")
|
||||
entity_id = str(item.get("id") or "")
|
||||
if not entity_type or not entity_id:
|
||||
continue
|
||||
await session.execute(
|
||||
delete(RawZenmoneyEntity).where(
|
||||
RawZenmoneyEntity.entity_type == entity_type,
|
||||
RawZenmoneyEntity.id == entity_id,
|
||||
)
|
||||
)
|
||||
stmt = pg_insert(RawZenmoneyDeletion).values(
|
||||
entity_type=entity_type,
|
||||
id=entity_id,
|
||||
stamp=item.get("stamp"),
|
||||
deleted_at=datetime.now(UTC),
|
||||
)
|
||||
stmt = stmt.on_conflict_do_update(
|
||||
index_elements=["entity_type", "id"],
|
||||
set_={"stamp": stmt.excluded.stamp, "deleted_at": stmt.excluded.deleted_at},
|
||||
)
|
||||
await session.execute(stmt)
|
||||
|
||||
if entity_type == "transaction":
|
||||
await session.execute(
|
||||
update(CashTxn)
|
||||
.where(CashTxn.source == SOURCE, CashTxn.source_id == entity_id)
|
||||
.values(deleted=True, flow_type=FlowType.deleted)
|
||||
)
|
||||
elif entity_type == "account":
|
||||
await session.execute(
|
||||
update(Account)
|
||||
.where(Account.source == SOURCE, Account.source_id == entity_id)
|
||||
.values(archived=True, include_in_net_worth=False)
|
||||
)
|
||||
elif entity_type == "tag":
|
||||
await session.execute(
|
||||
update(Category)
|
||||
.where(Category.source == SOURCE, Category.source_id == entity_id)
|
||||
.values(archived=True)
|
||||
)
|
||||
count += 1
|
||||
return count
|
||||
@@ -0,0 +1,39 @@
|
||||
"""Schedule: which source runs when (plan §2.4). Times are in the configured timezone."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
from zoneinfo import ZoneInfo
|
||||
|
||||
from apscheduler.triggers.base import BaseTrigger
|
||||
from apscheduler.triggers.combining import OrTrigger
|
||||
from apscheduler.triggers.cron import CronTrigger
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class JobSpec:
|
||||
source: str
|
||||
trigger: BaseTrigger
|
||||
"""A source appears at most once: `worker/scheduler.py` keys jobs by `sync:<source>`,
|
||||
so several times of day are one `OrTrigger`, not several specs."""
|
||||
|
||||
|
||||
MSK = ZoneInfo("Europe/Moscow")
|
||||
"""CBR publishes on Moscow time, so those jobs pin the zone instead of following settings."""
|
||||
|
||||
|
||||
def default_schedule() -> list[JobSpec]:
|
||||
# Each entry must name a registered source (see fintracker.sources).
|
||||
return [
|
||||
JobSpec("zenmoney", CronTrigger(minute="0,30")),
|
||||
# official rates appear around 13:30 MSK, the next business day's rate in the evening
|
||||
JobSpec(
|
||||
"cbr",
|
||||
OrTrigger(
|
||||
[
|
||||
CronTrigger(hour=13, minute=45, timezone=MSK),
|
||||
CronTrigger(hour=18, minute=0, timezone=MSK),
|
||||
]
|
||||
),
|
||||
),
|
||||
]
|
||||
@@ -0,0 +1,36 @@
|
||||
"""Postgres advisory locks so a scheduled run and a manual trigger never overlap."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import AsyncIterator
|
||||
from contextlib import asynccontextmanager
|
||||
|
||||
from sqlalchemy import text
|
||||
from sqlalchemy.ext.asyncio import AsyncConnection
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def try_advisory_lock(conn: AsyncConnection, key: str) -> AsyncIterator[bool]:
|
||||
"""Yield True if the session-level lock for `key` was acquired; release on exit."""
|
||||
got = (
|
||||
await conn.execute(text("SELECT pg_try_advisory_lock(hashtext(:k))"), {"k": key})
|
||||
).scalar_one()
|
||||
try:
|
||||
yield bool(got)
|
||||
finally:
|
||||
if got:
|
||||
await conn.execute(text("SELECT pg_advisory_unlock(hashtext(:k))"), {"k": key})
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def advisory_lock(conn: AsyncConnection, key: str) -> AsyncIterator[None]:
|
||||
"""Wait for the session-level lock for `key`, then release it on exit.
|
||||
|
||||
Blocking, unlike `try_advisory_lock`: used where the work must still happen after the
|
||||
current holder is done (a metric refresh queued behind a sync), not be skipped.
|
||||
"""
|
||||
await conn.execute(text("SELECT pg_advisory_lock(hashtext(:k))"), {"k": key})
|
||||
try:
|
||||
yield
|
||||
finally:
|
||||
await conn.execute(text("SELECT pg_advisory_unlock(hashtext(:k))"), {"k": key})
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Run one source sync end to end: lock, run log, cursor, error capture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from fintracker.config import get_settings
|
||||
from fintracker.db import get_engine, get_sessionmaker
|
||||
from fintracker.metrics.refresh import refresh_all
|
||||
from fintracker.models import RunStatus, SyncRun, SyncState
|
||||
from fintracker.sources import registry
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.worker.locks import try_advisory_lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Skipped(Exception):
|
||||
"""Another run of the same source holds the lock."""
|
||||
|
||||
|
||||
async def run_source(name: str, triggered_by: str) -> SyncRun:
|
||||
source = registry.get(name)
|
||||
settings = get_settings()
|
||||
# the lock lives on its own connection for the whole run
|
||||
async with (
|
||||
get_engine().connect() as lock_conn,
|
||||
try_advisory_lock(lock_conn, f"sync:{name}") as got,
|
||||
):
|
||||
if not got:
|
||||
raise Skipped(name)
|
||||
async with get_sessionmaker()() as session:
|
||||
state = await session.get(SyncState, name)
|
||||
if state is None:
|
||||
state = SyncState(source=name)
|
||||
session.add(state)
|
||||
run = SyncRun(
|
||||
source=name,
|
||||
status=RunStatus.running,
|
||||
triggered_by=triggered_by,
|
||||
cursor_before=state.cursor,
|
||||
)
|
||||
session.add(run)
|
||||
state.last_run_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
run_id = run.id
|
||||
|
||||
ctx = SyncContext(
|
||||
session=session,
|
||||
settings=settings,
|
||||
cursor_before=state.cursor,
|
||||
triggered_by=triggered_by,
|
||||
)
|
||||
try:
|
||||
result: SyncResult = await source.sync(ctx)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
log.exception("sync %s failed", name)
|
||||
async with get_sessionmaker()() as s2:
|
||||
run2 = await s2.get(SyncRun, run_id)
|
||||
assert run2 is not None
|
||||
run2.status = RunStatus.error
|
||||
run2.finished_at = datetime.now(UTC)
|
||||
run2.error = f"{type(exc).__name__}: {exc}\n{traceback.format_exc()[-4000:]}"
|
||||
await s2.commit()
|
||||
return run2
|
||||
|
||||
run = (await session.execute(select(SyncRun).where(SyncRun.id == run_id))).scalar_one()
|
||||
state = await session.get(SyncState, name)
|
||||
assert state is not None
|
||||
if result.cursor_after is not None:
|
||||
state.cursor = result.cursor_after
|
||||
state.last_success_at = datetime.now(UTC)
|
||||
run.status = RunStatus.ok
|
||||
run.finished_at = datetime.now(UTC)
|
||||
run.cursor_after = state.cursor
|
||||
run.counts = dict(result.counts)
|
||||
run.warnings = list(result.warnings)
|
||||
await session.commit()
|
||||
log.info("sync %s ok: %s", name, result.counts)
|
||||
if result.changed:
|
||||
entry = await refresh_all(session, trigger=f"sync:{name}")
|
||||
if entry.error:
|
||||
# the download itself succeeded, so the cursor stays advanced (a replay
|
||||
# would fix nothing); the run is still an error, because the metrics the
|
||||
# UI reads are now stale. `scheduler._finish_job` picks this up.
|
||||
run.status = RunStatus.error
|
||||
run.error = f"metrics refresh failed: {entry.error}"
|
||||
await session.commit()
|
||||
log.error("sync %s: metrics refresh failed", name)
|
||||
return run
|
||||
@@ -0,0 +1,118 @@
|
||||
"""Worker process: APScheduler for the timetable + a poller for manual `sync_job` rows."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import signal
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from apscheduler.schedulers.asyncio import AsyncIOScheduler
|
||||
from sqlalchemy import select
|
||||
|
||||
from fintracker.config import get_settings
|
||||
from fintracker.db import get_sessionmaker, reset_engine
|
||||
from fintracker.models import JobStatus, SyncJob
|
||||
from fintracker.sources import registry
|
||||
from fintracker.worker.jobs import default_schedule
|
||||
from fintracker.worker.runner import Skipped, run_source
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
POLL_SECONDS = 5
|
||||
|
||||
|
||||
async def _scheduled(source: str) -> None:
|
||||
try:
|
||||
await run_source(source, triggered_by="schedule")
|
||||
except Skipped:
|
||||
log.info("sync %s skipped: already running", source)
|
||||
|
||||
|
||||
async def _process_queued_jobs() -> None:
|
||||
async with get_sessionmaker()() as session:
|
||||
jobs = (
|
||||
(
|
||||
await session.execute(
|
||||
select(SyncJob)
|
||||
.where(SyncJob.status == JobStatus.queued)
|
||||
.order_by(SyncJob.requested_at)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
for job in jobs:
|
||||
if job.source not in registry.names():
|
||||
await _finish_job(job.id, JobStatus.error, error=f"unknown source {job.source}")
|
||||
continue
|
||||
await _mark_running(job.id)
|
||||
try:
|
||||
run = await run_source(job.source, triggered_by="manual")
|
||||
except Skipped:
|
||||
await _finish_job(job.id, JobStatus.error, error="already running")
|
||||
continue
|
||||
status = JobStatus.done if run.error is None else JobStatus.error
|
||||
await _finish_job(job.id, status, run_id=run.id, error=run.error)
|
||||
|
||||
|
||||
async def _mark_running(job_id) -> None:
|
||||
async with get_sessionmaker()() as session:
|
||||
job = await session.get(SyncJob, job_id)
|
||||
if job is not None:
|
||||
job.status = JobStatus.running
|
||||
job.started_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
|
||||
|
||||
async def _finish_job(job_id, status: JobStatus, *, run_id=None, error: str | None = None) -> None:
|
||||
async with get_sessionmaker()() as session:
|
||||
job = await session.get(SyncJob, job_id)
|
||||
if job is not None:
|
||||
job.status = status
|
||||
job.finished_at = datetime.now(UTC)
|
||||
job.run_id = run_id
|
||||
job.error = error
|
||||
await session.commit()
|
||||
|
||||
|
||||
def build_scheduler() -> AsyncIOScheduler:
|
||||
settings = get_settings()
|
||||
scheduler = AsyncIOScheduler(timezone=settings.timezone)
|
||||
for spec in default_schedule():
|
||||
scheduler.add_job(
|
||||
_scheduled,
|
||||
spec.trigger,
|
||||
args=[spec.source],
|
||||
id=f"sync:{spec.source}",
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
misfire_grace_time=600,
|
||||
)
|
||||
scheduler.add_job(
|
||||
_process_queued_jobs,
|
||||
"interval",
|
||||
seconds=POLL_SECONDS,
|
||||
id="poll-sync-jobs",
|
||||
max_instances=1,
|
||||
coalesce=True,
|
||||
)
|
||||
return scheduler
|
||||
|
||||
|
||||
async def serve_forever() -> None:
|
||||
scheduler = build_scheduler()
|
||||
stop = asyncio.Event()
|
||||
loop = asyncio.get_running_loop()
|
||||
for sig in (signal.SIGINT, signal.SIGTERM):
|
||||
loop.add_signal_handler(sig, stop.set)
|
||||
scheduler.start()
|
||||
log.info(
|
||||
"worker started; sources=%s jobs=%s", registry.names(), [j.id for j in scheduler.get_jobs()]
|
||||
)
|
||||
try:
|
||||
await stop.wait()
|
||||
finally:
|
||||
log.info("worker stopping")
|
||||
scheduler.shutdown(wait=True)
|
||||
await reset_engine()
|
||||
Reference in New Issue
Block a user