feat(sources): контракт источников, worker и синк ZenMoney + ЦБ
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
This commit is contained in:
Dmitry
@@ -0,0 +1,204 @@
|
||||
"""ZenMoney side: raw diff payloads, categories (tags), merchants, everyday transactions,
|
||||
hand-written rules and trips (plan §1.7)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class RawZenmoneyEntity(Base):
|
||||
"""Untouched JSON of every entity the /v8/diff endpoint returned; upserted by (type, id)."""
|
||||
|
||||
__tablename__ = "raw_zenmoney_entity"
|
||||
|
||||
entity_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
changed: Mapped[int | None] = mapped_column(BigInteger)
|
||||
payload: Mapped[dict[str, Any]]
|
||||
ingested_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawZenmoneyDeletion(Base):
|
||||
"""Deletions reported by diff; the raw row is removed, this keeps the fact for audit."""
|
||||
|
||||
__tablename__ = "raw_zenmoney_deletion"
|
||||
|
||||
entity_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
stamp: Mapped[int | None] = mapped_column(BigInteger)
|
||||
deleted_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class Category(TimestampMixin, Base):
|
||||
"""ZenMoney tag tree (max one level of nesting on their side)."""
|
||||
|
||||
__tablename__ = "category"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
parent_id: Mapped[int | None] = mapped_column(ForeignKey("category.id", ondelete="SET NULL"))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
icon: Mapped[str | None] = mapped_column(String(64))
|
||||
color: Mapped[int | None] = mapped_column(BigInteger)
|
||||
show_income: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
show_outcome: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
archived: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class Merchant(Base):
|
||||
__tablename__ = "merchant"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
|
||||
|
||||
class FlowType(enum.StrEnum):
|
||||
income = "income"
|
||||
expense = "expense"
|
||||
internal_transfer = "internal_transfer"
|
||||
savings_transfer = "savings_transfer"
|
||||
broker_external_flow = "broker_external_flow"
|
||||
deleted = "deleted"
|
||||
other = "other"
|
||||
|
||||
|
||||
class CashTxn(TimestampMixin, Base):
|
||||
"""One ZenMoney transaction. A transfer between own accounts is ONE row with both
|
||||
income and outcome > 0. Native amounts only — conversion happens in analytics."""
|
||||
|
||||
__tablename__ = "cash_txn"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
ts: Mapped[datetime]
|
||||
"""created timestamp from ZenMoney (UTC)."""
|
||||
date: Mapped[date] = mapped_column(index=True)
|
||||
income: Mapped[Decimal] = mapped_column(default=Decimal(0))
|
||||
income_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
income_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
outcome: Mapped[Decimal] = mapped_column(default=Decimal(0))
|
||||
outcome_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
outcome_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
op_income: Mapped[Decimal | None]
|
||||
op_income_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
op_outcome: Mapped[Decimal | None]
|
||||
op_outcome_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
payee: Mapped[str | None] = mapped_column(String(512))
|
||||
original_payee: Mapped[str | None] = mapped_column(String(512))
|
||||
merchant_id: Mapped[int | None] = mapped_column(ForeignKey("merchant.id", ondelete="SET NULL"))
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
mcc: Mapped[int | None] = mapped_column(Integer)
|
||||
hold: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
deleted: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
changed: Mapped[int | None] = mapped_column(BigInteger)
|
||||
primary_category_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("category.id", ondelete="SET NULL")
|
||||
)
|
||||
"""First tag as ZenMoney sent it (raw category)."""
|
||||
|
||||
# --- derived by analytics/classify (rules applied); rebuilt on every refresh ---
|
||||
flow_type: Mapped[FlowType] = mapped_column(
|
||||
db_enum(FlowType, "flow_type"), default=FlowType.other
|
||||
)
|
||||
category_id: Mapped[int | None] = mapped_column(ForeignKey("category.id", ondelete="SET NULL"))
|
||||
"""Effective category after rules (defaults to primary_category_id)."""
|
||||
payee_canonical: Mapped[str | None] = mapped_column(String(512))
|
||||
is_one_off: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
trip_id: Mapped[int | None] = mapped_column(ForeignKey("trip.id", ondelete="SET NULL"))
|
||||
meta: Mapped[dict[str, Any] | None]
|
||||
|
||||
|
||||
class CashTxnTag(Base):
|
||||
"""All tags of a transaction in ZenMoney order (ord 0 == primary)."""
|
||||
|
||||
__tablename__ = "cash_txn_tag"
|
||||
|
||||
txn_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("cash_txn.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
ord: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
category_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("category.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class RuleKind(enum.StrEnum):
|
||||
savings = "savings"
|
||||
"""expense -> savings_transfer (money moved into savings, not consumed)"""
|
||||
one_off = "one_off"
|
||||
"""still an expense, but out of the baseline and runway divisor"""
|
||||
category = "category"
|
||||
"""override category; `value` = category name"""
|
||||
payee = "payee"
|
||||
"""canonical payee name; `value` = the name"""
|
||||
broker_target = "broker_target"
|
||||
"""outflow funds a brokerage account; `value` = broker account id (str)"""
|
||||
ignore = "ignore"
|
||||
"""exclude from every metric: `flow_type` becomes `other`, and rule application STOPS for
|
||||
that transaction — no later rule (by priority) can put it back into a counted flow"""
|
||||
|
||||
|
||||
class RuleMatchType(enum.StrEnum):
|
||||
id = "id"
|
||||
payee = "payee"
|
||||
comment = "comment"
|
||||
category = "category"
|
||||
mcc = "mcc"
|
||||
account = "account"
|
||||
|
||||
|
||||
class Rule(TimestampMixin, Base):
|
||||
"""The judgement layer: ZenMoney knows what an operation *was*, rules say what it *meant*.
|
||||
`pattern` is matched case-insensitively with SQL LIKE semantics (`%` wildcards) except for
|
||||
`id`, `mcc` and `account`, which compare exactly."""
|
||||
|
||||
__tablename__ = "rule"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[RuleKind] = mapped_column(db_enum(RuleKind, "rule_kind"))
|
||||
match_type: Mapped[RuleMatchType] = mapped_column(db_enum(RuleMatchType, "rule_match_type"))
|
||||
pattern: Mapped[str] = mapped_column(String(512))
|
||||
value: Mapped[str | None] = mapped_column(String(512))
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
priority: Mapped[int] = mapped_column(Integer, default=100)
|
||||
last_matched_at: Mapped[datetime | None]
|
||||
match_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class Trip(TimestampMixin, Base):
|
||||
__tablename__ = "trip"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
date_from: Mapped[date]
|
||||
date_to: Mapped[date]
|
||||
country: Mapped[str | None] = mapped_column(String(2))
|
||||
Reference in New Issue
Block a user