feat(backend): каркас — конфиг, движок БД, базовые модели и первая миграция
SQLAlchemy 2 async на asyncpg, Alembic, pydantic-settings. Деньги везде NUMERIC(24,10) с колонкой валюты рядом (db/base.py), postgres-enum'ы хранят значения, а не имена, чтобы база читалась так же, как API. Первая миграция: app_user, account, portfolio, instrument, sync_run, sync_job. metrics/refresh.py задаёт порядок пересборки metric_* и сериализует параллельные пересчёты advisory-локом на отдельном соединении: каждый шаг заменяет свою таблицу целиком, два одновременных прогона затёрли бы друг друга.
This commit is contained in:
Dmitry
@@ -0,0 +1,136 @@
|
||||
"""ORM models. Importing this package registers every table on `Base.metadata`."""
|
||||
|
||||
from fintracker.models.accounts import (
|
||||
Account,
|
||||
AccountKind,
|
||||
AccountLink,
|
||||
AccountRole,
|
||||
Broker,
|
||||
EventSource,
|
||||
Portfolio,
|
||||
PortfolioAccount,
|
||||
)
|
||||
from fintracker.models.auth import AppUser, RefreshToken
|
||||
from fintracker.models.instruments import AssetClass, Instrument, InstrumentAlias
|
||||
from fintracker.models.ledger import (
|
||||
EXTERNAL_FLOW_KINDS,
|
||||
POSITION_KINDS,
|
||||
Event,
|
||||
EventKind,
|
||||
EventStatus,
|
||||
Lot,
|
||||
LotDisposal,
|
||||
)
|
||||
from fintracker.models.metrics import (
|
||||
MetricCashFlowMonthly,
|
||||
MetricDataQuality,
|
||||
MetricHolding,
|
||||
MetricNetWorthDaily,
|
||||
MetricPortfolioValueDaily,
|
||||
MetricRefreshLog,
|
||||
MetricReturns,
|
||||
MetricRunway,
|
||||
MetricSpendingByCategory,
|
||||
)
|
||||
from fintracker.models.pricing import (
|
||||
CashSnapshot,
|
||||
CorporateAction,
|
||||
CorporateActionKind,
|
||||
CorporateActionStatus,
|
||||
FxRateDaily,
|
||||
PositionSnapshot,
|
||||
PriceDaily,
|
||||
PriceLast,
|
||||
PriceManual,
|
||||
RawCbrRate,
|
||||
)
|
||||
from fintracker.models.sync import (
|
||||
JobStatus,
|
||||
RunStatus,
|
||||
SourceCredential,
|
||||
SyncJob,
|
||||
SyncRun,
|
||||
SyncState,
|
||||
)
|
||||
from fintracker.models.tinvest import (
|
||||
RawTinvestEvent,
|
||||
RawTinvestInstrument,
|
||||
RawTinvestOperation,
|
||||
RawTinvestSnapshot,
|
||||
)
|
||||
from fintracker.models.zenmoney import (
|
||||
CashTxn,
|
||||
CashTxnTag,
|
||||
Category,
|
||||
FlowType,
|
||||
Merchant,
|
||||
RawZenmoneyDeletion,
|
||||
RawZenmoneyEntity,
|
||||
Rule,
|
||||
RuleKind,
|
||||
RuleMatchType,
|
||||
Trip,
|
||||
)
|
||||
|
||||
__all__ = [
|
||||
"EXTERNAL_FLOW_KINDS",
|
||||
"POSITION_KINDS",
|
||||
"Account",
|
||||
"AccountKind",
|
||||
"AccountLink",
|
||||
"AccountRole",
|
||||
"AppUser",
|
||||
"AssetClass",
|
||||
"Broker",
|
||||
"CashSnapshot",
|
||||
"CashTxn",
|
||||
"CashTxnTag",
|
||||
"Category",
|
||||
"CorporateAction",
|
||||
"CorporateActionKind",
|
||||
"CorporateActionStatus",
|
||||
"Event",
|
||||
"EventKind",
|
||||
"EventSource",
|
||||
"EventStatus",
|
||||
"FlowType",
|
||||
"FxRateDaily",
|
||||
"Instrument",
|
||||
"InstrumentAlias",
|
||||
"JobStatus",
|
||||
"Lot",
|
||||
"LotDisposal",
|
||||
"Merchant",
|
||||
"MetricCashFlowMonthly",
|
||||
"MetricDataQuality",
|
||||
"MetricHolding",
|
||||
"MetricNetWorthDaily",
|
||||
"MetricPortfolioValueDaily",
|
||||
"MetricRefreshLog",
|
||||
"MetricReturns",
|
||||
"MetricRunway",
|
||||
"MetricSpendingByCategory",
|
||||
"Portfolio",
|
||||
"PortfolioAccount",
|
||||
"PositionSnapshot",
|
||||
"PriceDaily",
|
||||
"PriceLast",
|
||||
"PriceManual",
|
||||
"RawCbrRate",
|
||||
"RawTinvestEvent",
|
||||
"RawTinvestInstrument",
|
||||
"RawTinvestOperation",
|
||||
"RawTinvestSnapshot",
|
||||
"RawZenmoneyDeletion",
|
||||
"RawZenmoneyEntity",
|
||||
"RefreshToken",
|
||||
"Rule",
|
||||
"RuleKind",
|
||||
"RuleMatchType",
|
||||
"RunStatus",
|
||||
"SourceCredential",
|
||||
"SyncJob",
|
||||
"SyncRun",
|
||||
"SyncState",
|
||||
"Trip",
|
||||
]
|
||||
@@ -0,0 +1,116 @@
|
||||
"""Accounts, portfolios and the explicit ZenMoney <-> broker account mapping (plan §1.2)."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, String, UniqueConstraint
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class AccountKind(enum.StrEnum):
|
||||
zm_cash = "zm_cash"
|
||||
zm_card = "zm_card"
|
||||
zm_checking = "zm_checking"
|
||||
zm_deposit = "zm_deposit"
|
||||
zm_loan = "zm_loan"
|
||||
zm_emoney = "zm_emoney"
|
||||
zm_debt = "zm_debt"
|
||||
broker = "broker"
|
||||
manual_asset = "manual_asset"
|
||||
|
||||
|
||||
class AccountRole(enum.StrEnum):
|
||||
liquid = "liquid"
|
||||
savings = "savings"
|
||||
investment = "investment"
|
||||
debt = "debt"
|
||||
|
||||
|
||||
class Broker(enum.StrEnum):
|
||||
tinvest = "tinvest"
|
||||
sber = "sber"
|
||||
vtb = "vtb"
|
||||
other = "other"
|
||||
|
||||
|
||||
class EventSource(enum.StrEnum):
|
||||
"""Which feed is the truth for an account's ledger; others become `shadow` events."""
|
||||
|
||||
tinvest_api = "tinvest_api"
|
||||
report_sber = "report_sber"
|
||||
report_vtb = "report_vtb"
|
||||
manual = "manual"
|
||||
|
||||
|
||||
class Account(TimestampMixin, Base):
|
||||
__tablename__ = "account"
|
||||
__table_args__ = (UniqueConstraint("source", "source_id"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
kind: Mapped[AccountKind] = mapped_column(db_enum(AccountKind, "account_kind"))
|
||||
source: Mapped[str] = mapped_column(String(32))
|
||||
"""zenmoney | tinvest | report_sber | report_vtb | manual"""
|
||||
source_id: Mapped[str] = mapped_column(String(128))
|
||||
broker: Mapped[Broker | None] = mapped_column(db_enum(Broker, "broker"))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
include_in_net_worth: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
role: Mapped[AccountRole] = mapped_column(db_enum(AccountRole, "account_role"))
|
||||
mirror_of_account_id: Mapped[int | None] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="SET NULL")
|
||||
)
|
||||
"""ZenMoney account that merely mirrors a broker account (excluded from net worth)."""
|
||||
primary_event_source: Mapped[EventSource | None] = mapped_column(
|
||||
db_enum(EventSource, "event_source")
|
||||
)
|
||||
opened_at: Mapped[date | None]
|
||||
archived: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
deposit_terms: Mapped[dict[str, Any] | None]
|
||||
"""ZenMoney deposit/loan terms: percent, startDate, endDateOffset, capitalization…"""
|
||||
balance: Mapped[Decimal | None]
|
||||
"""Current balance as the source reports it (ZenMoney `balance`); native currency."""
|
||||
start_balance: Mapped[Decimal | None]
|
||||
credit_limit: Mapped[Decimal | None]
|
||||
balance_as_of: Mapped[datetime | None]
|
||||
"""When `balance` was observed (set by the source sync). Informational only: the
|
||||
net-worth walk in `analytics/networth.py` treats `balance` as the balance of TODAY
|
||||
(`today_local()`) and steps back by transaction DATES, not by this timestamp."""
|
||||
|
||||
|
||||
class Portfolio(TimestampMixin, Base):
|
||||
__tablename__ = "portfolio"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
name: Mapped[str] = mapped_column(String(128), unique=True)
|
||||
is_default: Mapped[bool] = mapped_column(Boolean, default=False)
|
||||
base_currency: Mapped[str] = mapped_column(String(3), default="RUB")
|
||||
|
||||
|
||||
class PortfolioAccount(Base):
|
||||
__tablename__ = "portfolio_account"
|
||||
|
||||
portfolio_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("portfolio.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
|
||||
|
||||
class AccountLink(Base):
|
||||
"""Explicit map used by the ZenMoney-transfer <-> broker-deposit matcher."""
|
||||
|
||||
__tablename__ = "account_link"
|
||||
|
||||
zm_account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), primary_key=True
|
||||
)
|
||||
broker_account_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("account.id", ondelete="CASCADE"), unique=True
|
||||
)
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
|
||||
from sqlalchemy import ForeignKey, String, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base
|
||||
|
||||
|
||||
class AppUser(Base):
|
||||
"""The single user of this deployment. Kept as a table so a second one is not a rewrite."""
|
||||
|
||||
__tablename__ = "app_user"
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
email: Mapped[str] = mapped_column(String(320), unique=True)
|
||||
password_hash: Mapped[str] = mapped_column(String(512))
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
|
||||
|
||||
class RefreshToken(Base):
|
||||
"""Refresh tokens are stored hashed so logout/revocation is real."""
|
||||
|
||||
__tablename__ = "refresh_token"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
user_id: Mapped[int] = mapped_column(ForeignKey("app_user.id", ondelete="CASCADE"), index=True)
|
||||
token_hash: Mapped[str] = mapped_column(String(64), unique=True)
|
||||
expires_at: Mapped[datetime]
|
||||
revoked_at: Mapped[datetime | None]
|
||||
created_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Instrument master (plan §1.3).
|
||||
|
||||
Identity resolution order: ISIN -> FIGI -> tinvest_uid -> (ticker, board) -> alias.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
from datetime import date
|
||||
from decimal import Decimal
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import Boolean, ForeignKey, Index, Integer, String, UniqueConstraint, text
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, TimestampMixin, db_enum
|
||||
|
||||
|
||||
class AssetClass(enum.StrEnum):
|
||||
share = "share"
|
||||
bond = "bond"
|
||||
etf = "etf"
|
||||
fund = "fund"
|
||||
currency = "currency"
|
||||
market_index = "index"
|
||||
deposit = "deposit"
|
||||
real_estate = "real_estate"
|
||||
crypto = "crypto"
|
||||
custom = "custom"
|
||||
|
||||
|
||||
class Instrument(TimestampMixin, Base):
|
||||
__tablename__ = "instrument"
|
||||
__table_args__ = (
|
||||
Index("uq_instrument_isin", "isin", unique=True, postgresql_where=text("isin IS NOT NULL")),
|
||||
Index("uq_instrument_figi", "figi", unique=True, postgresql_where=text("figi IS NOT NULL")),
|
||||
Index(
|
||||
"uq_instrument_tinvest_uid",
|
||||
"tinvest_uid",
|
||||
unique=True,
|
||||
postgresql_where=text("tinvest_uid IS NOT NULL"),
|
||||
),
|
||||
Index(
|
||||
"uq_instrument_ticker_board",
|
||||
"ticker",
|
||||
"board",
|
||||
unique=True,
|
||||
postgresql_where=text("ticker IS NOT NULL AND board IS NOT NULL"),
|
||||
),
|
||||
)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
asset_class: Mapped[AssetClass] = mapped_column(db_enum(AssetClass, "asset_class"))
|
||||
isin: Mapped[str | None] = mapped_column(String(12))
|
||||
figi: Mapped[str | None] = mapped_column(String(12))
|
||||
tinvest_uid: Mapped[str | None] = mapped_column(String(64))
|
||||
ticker: Mapped[str | None] = mapped_column(String(32))
|
||||
board: Mapped[str | None] = mapped_column(String(16))
|
||||
"""MOEX board / T-Invest class_code, e.g. TQBR, TQOB."""
|
||||
exchange: Mapped[str | None] = mapped_column(String(32))
|
||||
name: Mapped[str] = mapped_column(String(256))
|
||||
issuer: Mapped[str | None] = mapped_column(String(256))
|
||||
currency: Mapped[str] = mapped_column(String(3))
|
||||
lot: Mapped[int] = mapped_column(Integer, default=1)
|
||||
nominal: Mapped[Decimal | None]
|
||||
nominal_currency: Mapped[str | None] = mapped_column(String(3))
|
||||
maturity_date: Mapped[date | None]
|
||||
sector: Mapped[str | None] = mapped_column(String(64))
|
||||
country: Mapped[str | None] = mapped_column(String(2))
|
||||
is_active: Mapped[bool] = mapped_column(Boolean, default=True)
|
||||
meta: Mapped[dict[str, Any] | None]
|
||||
|
||||
|
||||
class InstrumentAlias(Base):
|
||||
"""Source-specific keys resolving to an instrument, e.g. ('report_vtb', 'NAME:Газпром ао')."""
|
||||
|
||||
__tablename__ = "instrument_alias"
|
||||
__table_args__ = (UniqueConstraint("source", "source_key"),)
|
||||
|
||||
id: Mapped[int] = mapped_column(primary_key=True)
|
||||
instrument_id: Mapped[int] = mapped_column(
|
||||
ForeignKey("instrument.id", ondelete="CASCADE"), index=True
|
||||
)
|
||||
source: Mapped[str] = mapped_column(String(32))
|
||||
source_key: Mapped[str] = mapped_column(String(256))
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Sync bookkeeping: cursors, run log, manual job queue, per-source credentials."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import enum
|
||||
import uuid
|
||||
from datetime import datetime
|
||||
from typing import Any
|
||||
|
||||
from sqlalchemy import ForeignKey, String, Text, func
|
||||
from sqlalchemy.orm import Mapped, mapped_column
|
||||
|
||||
from fintracker.db.base import Base, db_enum
|
||||
|
||||
|
||||
class RunStatus(enum.StrEnum):
|
||||
running = "running"
|
||||
ok = "ok"
|
||||
error = "error"
|
||||
|
||||
|
||||
class JobStatus(enum.StrEnum):
|
||||
queued = "queued"
|
||||
running = "running"
|
||||
done = "done"
|
||||
error = "error"
|
||||
|
||||
|
||||
class SyncState(Base):
|
||||
"""One row per source: where the incremental sync left off."""
|
||||
|
||||
__tablename__ = "sync_state"
|
||||
|
||||
source: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
cursor: Mapped[str | None] = mapped_column(Text)
|
||||
last_run_at: Mapped[datetime | None]
|
||||
last_success_at: Mapped[datetime | None]
|
||||
|
||||
|
||||
class SyncRun(Base):
|
||||
__tablename__ = "sync_run"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
source: Mapped[str] = mapped_column(String(64), index=True)
|
||||
status: Mapped[RunStatus] = mapped_column(db_enum(RunStatus, "run_status"))
|
||||
triggered_by: Mapped[str] = mapped_column(String(32))
|
||||
"""schedule | manual | cli"""
|
||||
started_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
finished_at: Mapped[datetime | None]
|
||||
cursor_before: Mapped[str | None] = mapped_column(Text)
|
||||
cursor_after: Mapped[str | None] = mapped_column(Text)
|
||||
counts: Mapped[dict[str, Any] | None]
|
||||
warnings: Mapped[list[Any] | None]
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class SyncJob(Base):
|
||||
"""Manual trigger queue: API inserts, worker picks up (plan §2.4)."""
|
||||
|
||||
__tablename__ = "sync_job"
|
||||
|
||||
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
||||
source: Mapped[str] = mapped_column(String(64), index=True)
|
||||
status: Mapped[JobStatus] = mapped_column(db_enum(JobStatus, "job_status"), index=True)
|
||||
requested_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
||||
started_at: Mapped[datetime | None]
|
||||
finished_at: Mapped[datetime | None]
|
||||
run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("sync_run.id", ondelete="SET NULL"))
|
||||
error: Mapped[str | None] = mapped_column(Text)
|
||||
|
||||
|
||||
class SourceCredential(Base):
|
||||
"""Rotating credentials a source manages itself (e.g. ZenMoney refresh token).
|
||||
|
||||
Static tokens still come from the environment; this is for what the worker must
|
||||
be able to update unattended.
|
||||
"""
|
||||
|
||||
__tablename__ = "source_credential"
|
||||
|
||||
source: Mapped[str] = mapped_column(String(64), primary_key=True)
|
||||
payload: Mapped[dict[str, Any]]
|
||||
updated_at: Mapped[datetime] = mapped_column(server_default=func.now(), onupdate=func.now())
|
||||
Reference in New Issue
Block a user