feat(sources): контракт источников, worker и синк ZenMoney + ЦБ
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
This commit is contained in:
Dmitry
@@ -0,0 +1,168 @@
|
||||
"""FX rates. Later phases add price_daily / price_last / corporate_action here (plan §1.6)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, UniqueConstraint, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class RawCbrRate(Base):
|
||||
"""CBR official rate as published: RUB per `nominal` units of `ccy`, on business days only."""
|
||||
|
||||
__tablename__ = "raw_cbr_rate"
|
||||
|
||||
rate_date: Mapped[date] = mapped_column(primary_key=True)
|
||||
ccy: Mapped[str] = mapped_column(String(3), primary_key=True)
|
||||
nominal: Mapped[int] = mapped_column(Integer, default=1)
|
||||
value: Mapped[Decimal]
|
||||
"""RUB per `nominal` units, exactly as CBR printed it."""
|
||||
fetched_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class FxRateDaily(Base):
|
||||
"""Every calendar day has a rate: business-day quotes carried forward (and backward before
|
||||
the first quote). RUB is present as 1.0. Rebuilt by pricing/fx.py on each refresh."""
|
||||
|
||||
__tablename__ = "fx_rate_daily"
|
||||
|
||||
d: Mapped[date] = mapped_column(primary_key=True)
|
||||
ccy: Mapped[str] = mapped_column(String(3), primary_key=True)
|
||||
rate_rub: Mapped[Decimal]
|
||||
"""RUB per ONE unit of ccy."""
|
||||
source: Mapped[str] = mapped_column(String(16), default="cbr")
|
||||
is_carried: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class PriceDaily(Base):
|
||||
"""Daily close per instrument. Bonds quote in percent of nominal, so `price_pct` carries
|
||||
the quote as published and `close` is the money value it resolves to."""
|
||||
|
||||
__tablename__ = "price_daily"
|
||||
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
d: Mapped[date] = mapped_column(primary_key=True)
|
||||
close: Mapped[Decimal]
|
||||
open: Mapped[Decimal | None]
|
||||
high: Mapped[Decimal | None]
|
||||
low: Mapped[Decimal | None]
|
||||
volume: Mapped[Decimal | None]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
"""tinvest | moex | manual"""
|
||||
price_pct: Mapped[Decimal | None]
|
||||
"""Bond quote in percent of nominal, as the exchange published it."""
|
||||
accrued_interest: Mapped[Decimal | None]
|
||||
|
||||
|
||||
class PriceLast(Base):
|
||||
"""Latest known price per instrument — what the positions screen shows between refreshes."""
|
||||
|
||||
__tablename__ = "price_last"
|
||||
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
ts: Mapped[datetime]
|
||||
price: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
|
||||
|
||||
class PriceManual(Base):
|
||||
"""User-set prices for what no exchange quotes: real estate, crypto, custom holdings."""
|
||||
|
||||
__tablename__ = "price_manual"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
d: Mapped[date]
|
||||
price: Mapped[Decimal]
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
note: Mapped[str | None] = mapped_column(String(256))
|
||||
|
||||
|
||||
class CorporateActionKind(enum.StrEnum):
|
||||
dividend = "dividend"
|
||||
coupon = "coupon"
|
||||
amortization = "amortization"
|
||||
repayment = "repayment"
|
||||
stock_split = "split"
|
||||
offer = "offer"
|
||||
|
||||
|
||||
class CorporateActionStatus(enum.StrEnum):
|
||||
forecast = "forecast"
|
||||
"""Projected from history — never mixed with announced money in the paid totals."""
|
||||
announced = "announced"
|
||||
paid = "paid"
|
||||
cancelled = "cancelled"
|
||||
|
||||
|
||||
class CorporateAction(TimestampMixin, Base):
|
||||
"""Declared and projected payouts, the source of the dividend/coupon calendar."""
|
||||
|
||||
__tablename__ = "corporate_action"
|
||||
__table_args__ = (
|
||||
UniqueConstraint("instrument_id", "kind", "source", "source_id"),
|
||||
Index("ix_corporate_action_pay_date", "pay_date"),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
instrument_id: Mapped[int] = mapped_column(ForeignKey("instrument.id", ondelete="CASCADE"))
|
||||
kind: Mapped[CorporateActionKind] = mapped_column(
|
||||
db_enum(CorporateActionKind, "corporate_action_kind")
|
||||
)
|
||||
status: Mapped[CorporateActionStatus] = mapped_column(
|
||||
db_enum(CorporateActionStatus, "corporate_action_status")
|
||||
)
|
||||
record_date: Mapped[date | None]
|
||||
ex_date: Mapped[date | None]
|
||||
pay_date: Mapped[date | None]
|
||||
amount_per_unit: Mapped[Decimal | None]
|
||||
currency: Mapped[str | None] = mapped_column(String(3))
|
||||
ratio: Mapped[Decimal | None]
|
||||
"""Split ratio; NULL for cash events."""
|
||||
source: Mapped[str] = mapped_column(String(16))
|
||||
source_id: Mapped[str | None] = mapped_column(String(128))
|
||||
|
||||
|
||||
class PositionSnapshot(Base):
|
||||
"""What the broker says a position is — kept for reconciliation, never for analytics."""
|
||||
|
||||
__tablename__ = "position_snapshot"
|
||||
|
||||
account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
as_of: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
qty: Mapped[Decimal]
|
||||
avg_price: Mapped[Decimal | None]
|
||||
market_value: Mapped[Decimal | None]
|
||||
currency: Mapped[str | None] = mapped_column(String(3))
|
||||
|
||||
|
||||
class CashSnapshot(Base):
|
||||
"""Broker-reported cash per currency — the counterpart of `position_snapshot`."""
|
||||
|
||||
__tablename__ = "cash_snapshot"
|
||||
|
||||
account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
currency: Mapped[str] = mapped_column(String(3), primary_key=True)
|
||||
as_of: Mapped[datetime] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(16), primary_key=True)
|
||||
balance: Mapped[Decimal]
|
||||
blocked: Mapped[Decimal | None]
|
||||
@@ -0,0 +1,204 @@
|
||||
"""ZenMoney side: raw diff payloads, categories (tags), merchants, everyday transactions,
|
||||
hand-written rules and trips (plan §1.7)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import (
|
||||
BigInteger,
|
||||
Boolean,
|
||||
ForeignKey,
|
||||
Integer,
|
||||
String,
|
||||
Text,
|
||||
UniqueConstraint,
|
||||
func,
|
||||
)
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class RawZenmoneyEntity(Base):
|
||||
"""Untouched JSON of every entity the /v8/diff endpoint returned; upserted by (type, id)."""
|
||||
|
||||
__tablename__ = "raw_zenmoney_entity"
|
||||
|
||||
entity_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
changed: Mapped[int | None] = mapped_column(BigInteger)
|
||||
payload: Mapped[dict[str, Any]]
|
||||
ingested_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RawZenmoneyDeletion(Base):
|
||||
"""Deletions reported by diff; the raw row is removed, this keeps the fact for audit."""
|
||||
|
||||
__tablename__ = "raw_zenmoney_deletion"
|
||||
|
||||
entity_type: Mapped[str] = mapped_column(String(32), primary_key=True)
|
||||
id: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
stamp: Mapped[int | None] = mapped_column(BigInteger)
|
||||
deleted_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class Category(TimestampMixin, Base):
|
||||
"""ZenMoney tag tree (max one level of nesting on their side)."""
|
||||
|
||||
__tablename__ = "category"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
parent_id: Mapped[int | None] = mapped_column(ForeignKey("category.id", ondelete="SET NULL"))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
icon: Mapped[str | None] = mapped_column(String(64))
|
||||
color: Mapped[int | None] = mapped_column(BigInteger)
|
||||
show_income: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
show_outcome: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
archived: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
|
||||
|
||||
class Merchant(Base):
|
||||
__tablename__ = "merchant"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
|
||||
|
||||
class FlowType(enum.StrEnum):
|
||||
income = "income"
|
||||
expense = "expense"
|
||||
internal_transfer = "internal_transfer"
|
||||
savings_transfer = "savings_transfer"
|
||||
broker_external_flow = "broker_external_flow"
|
||||
deleted = "deleted"
|
||||
other = "other"
|
||||
|
||||
|
||||
class CashTxn(TimestampMixin, Base):
|
||||
"""One ZenMoney transaction. A transfer between own accounts is ONE row with both
|
||||
income and outcome > 0. Native amounts only — conversion happens in analytics."""
|
||||
|
||||
__tablename__ = "cash_txn"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
source: Mapped[str] = mapped_column(String(32), default="zenmoney")
|
||||
source_id: Mapped[str] = mapped_column(String(64))
|
||||
ts: Mapped[datetime]
|
||||
"""created timestamp from ZenMoney (UTC)."""
|
||||
date: Mapped[date] = mapped_column(index=True)
|
||||
income: Mapped[Decimal] = mapped_column(default=Decimal(0))
|
||||
income_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
income_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
outcome: Mapped[Decimal] = mapped_column(default=Decimal(0))
|
||||
outcome_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
outcome_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL"), index=True
|
||||
)
|
||||
op_income: Mapped[Decimal | None]
|
||||
op_income_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
op_outcome: Mapped[Decimal | None]
|
||||
op_outcome_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
payee: Mapped[str | None] = mapped_column(String(512))
|
||||
original_payee: Mapped[str | None] = mapped_column(String(512))
|
||||
merchant_id: Mapped[int | None] = mapped_column(ForeignKey("merchant.id", ondelete="SET NULL"))
|
||||
comment: Mapped[str | None] = mapped_column(Text)
|
||||
mcc: Mapped[int | None] = mapped_column(Integer)
|
||||
hold: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
deleted: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
changed: Mapped[int | None] = mapped_column(BigInteger)
|
||||
primary_category_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("category.id", ondelete="SET NULL")
|
||||
)
|
||||
"""First tag as ZenMoney sent it (raw category)."""
|
||||
|
||||
# --- derived by analytics/classify (rules applied); rebuilt on every refresh ---
|
||||
flow_type: Mapped[FlowType] = mapped_column(
|
||||
db_enum(FlowType, "flow_type"), default=FlowType.other
|
||||
)
|
||||
category_id: Mapped[int | None] = mapped_column(ForeignKey("category.id", ondelete="SET NULL"))
|
||||
"""Effective category after rules (defaults to primary_category_id)."""
|
||||
payee_canonical: Mapped[str | None] = mapped_column(String(512))
|
||||
is_one_off: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
trip_id: Mapped[int | None] = mapped_column(ForeignKey("trip.id", ondelete="SET NULL"))
|
||||
meta: Mapped[dict[str, Any] | None]
|
||||
|
||||
|
||||
class CashTxnTag(Base):
|
||||
"""All tags of a transaction in ZenMoney order (ord 0 == primary)."""
|
||||
|
||||
__tablename__ = "cash_txn_tag"
|
||||
|
||||
txn_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("cash_txn.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
ord: Mapped[int] = mapped_column(Integer, primary_key=True)
|
||||
category_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("category.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
|
||||
|
||||
class RuleKind(enum.StrEnum):
|
||||
savings = "savings"
|
||||
"""expense -> savings_transfer (money moved into savings, not consumed)"""
|
||||
one_off = "one_off"
|
||||
"""still an expense, but out of the baseline and runway divisor"""
|
||||
category = "category"
|
||||
"""override category; `value` = category name"""
|
||||
payee = "payee"
|
||||
"""canonical payee name; `value` = the name"""
|
||||
broker_target = "broker_target"
|
||||
"""outflow funds a brokerage account; `value` = broker account id (str)"""
|
||||
ignore = "ignore"
|
||||
"""exclude from every metric: `flow_type` becomes `other`, and rule application STOPS for
|
||||
that transaction — no later rule (by priority) can put it back into a counted flow"""
|
||||
|
||||
|
||||
class RuleMatchType(enum.StrEnum):
|
||||
id = "id"
|
||||
payee = "payee"
|
||||
comment = "comment"
|
||||
category = "category"
|
||||
mcc = "mcc"
|
||||
account = "account"
|
||||
|
||||
|
||||
class Rule(TimestampMixin, Base):
|
||||
"""The judgement layer: ZenMoney knows what an operation *was*, rules say what it *meant*.
|
||||
`pattern` is matched case-insensitively with SQL LIKE semantics (`%` wildcards) except for
|
||||
`id`, `mcc` and `account`, which compare exactly."""
|
||||
|
||||
__tablename__ = "rule"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[RuleKind] = mapped_column(db_enum(RuleKind, "rule_kind"))
|
||||
match_type: Mapped[RuleMatchType] = mapped_column(db_enum(RuleMatchType, "rule_match_type"))
|
||||
pattern: Mapped[str] = mapped_column(String(512))
|
||||
value: Mapped[str | None] = mapped_column(String(512))
|
||||
note: Mapped[str | None] = mapped_column(Text)
|
||||
enabled: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
priority: Mapped[int] = mapped_column(Integer, default=100)
|
||||
last_matched_at: Mapped[datetime | None]
|
||||
match_count: Mapped[int] = mapped_column(Integer, default=0)
|
||||
|
||||
|
||||
class Trip(TimestampMixin, Base):
|
||||
__tablename__ = "trip"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
date_from: Mapped[date]
|
||||
date_to: Mapped[date]
|
||||
country: Mapped[str | None] = mapped_column(String(2))
|
||||
Reference in New Issue
Block a user