Dmitry
c55fe19e48
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
459 lines
18 KiB
Python
459 lines
18 KiB
Python
"""phase1 zenmoney fx metrics
|
|
|
|
Revision ID: 0506c15002c8
|
|
Revises: 05beabc436ad
|
|
Create Date: 2026-09-17 22:38:27.588642
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
from collections.abc import Sequence
|
|
|
|
import sqlalchemy as sa
|
|
from alembic import op
|
|
from sqlalchemy.dialects import postgresql
|
|
|
|
revision: str = "0506c15002c8"
|
|
down_revision: str | None = "05beabc436ad"
|
|
branch_labels: str | Sequence[str] | None = None
|
|
depends_on: str | Sequence[str] | None = None
|
|
|
|
|
|
def upgrade() -> None:
|
|
# ### commands auto generated by Alembic - please adjust! ###
|
|
op.create_table(
|
|
"category",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("source", sa.String(length=32), nullable=False),
|
|
sa.Column("source_id", sa.String(length=64), nullable=False),
|
|
sa.Column("parent_id", sa.Integer(), nullable=True),
|
|
sa.Column("name", sa.String(length=256), nullable=False),
|
|
sa.Column("icon", sa.String(length=64), nullable=True),
|
|
sa.Column("color", sa.BigInteger(), nullable=True),
|
|
sa.Column("show_income", sa.Boolean(), nullable=False),
|
|
sa.Column("show_outcome", sa.Boolean(), nullable=False),
|
|
sa.Column("archived", sa.Boolean(), nullable=False),
|
|
sa.Column(
|
|
"created_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"updated_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["parent_id"],
|
|
["category.id"],
|
|
name=op.f("fk_category_parent_id_category"),
|
|
ondelete="SET NULL",
|
|
),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_category")),
|
|
sa.UniqueConstraint("source", "source_id", name=op.f("uq_category_source_source_id")),
|
|
)
|
|
op.create_table(
|
|
"fx_rate_daily",
|
|
sa.Column("d", sa.Date(), nullable=False),
|
|
sa.Column("ccy", sa.String(length=3), nullable=False),
|
|
sa.Column("rate_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("source", sa.String(length=16), nullable=False),
|
|
sa.Column("is_carried", sa.Boolean(), nullable=False),
|
|
sa.PrimaryKeyConstraint("d", "ccy", name=op.f("pk_fx_rate_daily")),
|
|
)
|
|
op.create_table(
|
|
"merchant",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("source", sa.String(length=32), nullable=False),
|
|
sa.Column("source_id", sa.String(length=64), nullable=False),
|
|
sa.Column("name", sa.String(length=256), nullable=False),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_merchant")),
|
|
sa.UniqueConstraint("source", "source_id", name=op.f("uq_merchant_source_source_id")),
|
|
)
|
|
op.create_table(
|
|
"metric_cash_flow_monthly",
|
|
sa.Column("month", sa.Date(), nullable=False),
|
|
sa.Column("income_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("expense_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("baseline_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("one_off_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("savings_transfer_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("savings_rate", sa.Numeric(precision=24, scale=10), nullable=True),
|
|
sa.Column("txn_count", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"computed_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("month", name=op.f("pk_metric_cash_flow_monthly")),
|
|
)
|
|
op.create_table(
|
|
"metric_data_quality",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("check_name", sa.String(length=64), nullable=False),
|
|
sa.Column("severity", sa.String(length=8), nullable=False),
|
|
sa.Column("detail", sa.Text(), nullable=False),
|
|
sa.Column("count", sa.Integer(), nullable=False),
|
|
sa.Column("ref", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
|
sa.Column(
|
|
"computed_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_data_quality")),
|
|
)
|
|
op.create_index(
|
|
op.f("ix_metric_data_quality_check_name"),
|
|
"metric_data_quality",
|
|
["check_name"],
|
|
unique=False,
|
|
)
|
|
op.create_table(
|
|
"metric_net_worth_daily",
|
|
sa.Column("d", sa.Date(), nullable=False),
|
|
sa.Column("total_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("liquid_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("savings_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("investment_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("debt_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("by_currency", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
|
sa.Column("missing_fx_count", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"computed_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("d", name=op.f("pk_metric_net_worth_daily")),
|
|
)
|
|
op.create_table(
|
|
"metric_refresh_log",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"started_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("finished_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("trigger", sa.String(length=32), nullable=False),
|
|
sa.Column("error", sa.Text(), nullable=True),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_refresh_log")),
|
|
)
|
|
op.create_table(
|
|
"metric_runway",
|
|
sa.Column("as_of", sa.Date(), nullable=False),
|
|
sa.Column("liquid_reserve_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("avg_baseline_3m_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("runway_months", sa.Numeric(precision=24, scale=10), nullable=True),
|
|
sa.Column(
|
|
"computed_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("as_of", name=op.f("pk_metric_runway")),
|
|
)
|
|
op.create_table(
|
|
"raw_cbr_rate",
|
|
sa.Column("rate_date", sa.Date(), nullable=False),
|
|
sa.Column("ccy", sa.String(length=3), nullable=False),
|
|
sa.Column("nominal", sa.Integer(), nullable=False),
|
|
sa.Column("value", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column(
|
|
"fetched_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("rate_date", "ccy", name=op.f("pk_raw_cbr_rate")),
|
|
)
|
|
op.create_table(
|
|
"raw_zenmoney_deletion",
|
|
sa.Column("entity_type", sa.String(length=32), nullable=False),
|
|
sa.Column("id", sa.String(length=64), nullable=False),
|
|
sa.Column("stamp", sa.BigInteger(), nullable=True),
|
|
sa.Column(
|
|
"deleted_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("entity_type", "id", name=op.f("pk_raw_zenmoney_deletion")),
|
|
)
|
|
op.create_table(
|
|
"raw_zenmoney_entity",
|
|
sa.Column("entity_type", sa.String(length=32), nullable=False),
|
|
sa.Column("id", sa.String(length=64), nullable=False),
|
|
sa.Column("changed", sa.BigInteger(), nullable=True),
|
|
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
|
sa.Column(
|
|
"ingested_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("entity_type", "id", name=op.f("pk_raw_zenmoney_entity")),
|
|
)
|
|
op.create_table(
|
|
"rule",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"kind",
|
|
sa.Enum(
|
|
"savings",
|
|
"one_off",
|
|
"category",
|
|
"payee",
|
|
"broker_target",
|
|
"ignore",
|
|
name="rule_kind",
|
|
),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"match_type",
|
|
sa.Enum("id", "payee", "comment", "category", "mcc", "account", name="rule_match_type"),
|
|
nullable=False,
|
|
),
|
|
sa.Column("pattern", sa.String(length=512), nullable=False),
|
|
sa.Column("value", sa.String(length=512), nullable=True),
|
|
sa.Column("note", sa.Text(), nullable=True),
|
|
sa.Column("enabled", sa.Boolean(), nullable=False),
|
|
sa.Column("priority", sa.Integer(), nullable=False),
|
|
sa.Column("last_matched_at", sa.DateTime(timezone=True), nullable=True),
|
|
sa.Column("match_count", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"created_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"updated_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_rule")),
|
|
)
|
|
op.create_table(
|
|
"trip",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("name", sa.String(length=256), nullable=False),
|
|
sa.Column("date_from", sa.Date(), nullable=False),
|
|
sa.Column("date_to", sa.Date(), nullable=False),
|
|
sa.Column("country", sa.String(length=2), nullable=True),
|
|
sa.Column(
|
|
"created_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"updated_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_trip")),
|
|
)
|
|
op.create_table(
|
|
"cash_txn",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("source", sa.String(length=32), nullable=False),
|
|
sa.Column("source_id", sa.String(length=64), nullable=False),
|
|
sa.Column("ts", sa.DateTime(timezone=True), nullable=False),
|
|
sa.Column("date", sa.Date(), nullable=False),
|
|
sa.Column("income", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("income_currency", sa.String(length=3), nullable=True),
|
|
sa.Column("income_account_id", sa.Integer(), nullable=True),
|
|
sa.Column("outcome", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("outcome_currency", sa.String(length=3), nullable=True),
|
|
sa.Column("outcome_account_id", sa.Integer(), nullable=True),
|
|
sa.Column("op_income", sa.Numeric(precision=24, scale=10), nullable=True),
|
|
sa.Column("op_income_currency", sa.String(length=3), nullable=True),
|
|
sa.Column("op_outcome", sa.Numeric(precision=24, scale=10), nullable=True),
|
|
sa.Column("op_outcome_currency", sa.String(length=3), nullable=True),
|
|
sa.Column("payee", sa.String(length=512), nullable=True),
|
|
sa.Column("original_payee", sa.String(length=512), nullable=True),
|
|
sa.Column("merchant_id", sa.Integer(), nullable=True),
|
|
sa.Column("comment", sa.Text(), nullable=True),
|
|
sa.Column("mcc", sa.Integer(), nullable=True),
|
|
sa.Column("hold", sa.Boolean(), nullable=False),
|
|
sa.Column("deleted", sa.Boolean(), nullable=False),
|
|
sa.Column("changed", sa.BigInteger(), nullable=True),
|
|
sa.Column("primary_category_id", sa.Integer(), nullable=True),
|
|
sa.Column(
|
|
"flow_type",
|
|
sa.Enum(
|
|
"income",
|
|
"expense",
|
|
"internal_transfer",
|
|
"savings_transfer",
|
|
"broker_external_flow",
|
|
"deleted",
|
|
"other",
|
|
name="flow_type",
|
|
),
|
|
nullable=False,
|
|
),
|
|
sa.Column("category_id", sa.Integer(), nullable=True),
|
|
sa.Column("payee_canonical", sa.String(length=512), nullable=True),
|
|
sa.Column("is_one_off", sa.Boolean(), nullable=False),
|
|
sa.Column("trip_id", sa.Integer(), nullable=True),
|
|
sa.Column("meta", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
|
sa.Column(
|
|
"created_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.Column(
|
|
"updated_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["category_id"],
|
|
["category.id"],
|
|
name=op.f("fk_cash_txn_category_id_category"),
|
|
ondelete="SET NULL",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["income_account_id"],
|
|
["account.id"],
|
|
name=op.f("fk_cash_txn_income_account_id_account"),
|
|
ondelete="SET NULL",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["merchant_id"],
|
|
["merchant.id"],
|
|
name=op.f("fk_cash_txn_merchant_id_merchant"),
|
|
ondelete="SET NULL",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["outcome_account_id"],
|
|
["account.id"],
|
|
name=op.f("fk_cash_txn_outcome_account_id_account"),
|
|
ondelete="SET NULL",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["primary_category_id"],
|
|
["category.id"],
|
|
name=op.f("fk_cash_txn_primary_category_id_category"),
|
|
ondelete="SET NULL",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["trip_id"], ["trip.id"], name=op.f("fk_cash_txn_trip_id_trip"), ondelete="SET NULL"
|
|
),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_cash_txn")),
|
|
sa.UniqueConstraint("source", "source_id", name=op.f("uq_cash_txn_source_source_id")),
|
|
)
|
|
op.create_index(op.f("ix_cash_txn_date"), "cash_txn", ["date"], unique=False)
|
|
op.create_index(
|
|
op.f("ix_cash_txn_income_account_id"), "cash_txn", ["income_account_id"], unique=False
|
|
)
|
|
op.create_index(
|
|
op.f("ix_cash_txn_outcome_account_id"), "cash_txn", ["outcome_account_id"], unique=False
|
|
)
|
|
op.create_table(
|
|
"metric_spending_by_category",
|
|
sa.Column("id", sa.Integer(), nullable=False),
|
|
sa.Column("month", sa.Date(), nullable=False),
|
|
sa.Column("category_id", sa.Integer(), nullable=True),
|
|
sa.Column("root_category_id", sa.Integer(), nullable=True),
|
|
sa.Column("amount_rub", sa.Numeric(precision=24, scale=10), nullable=False),
|
|
sa.Column("txn_count", sa.Integer(), nullable=False),
|
|
sa.Column(
|
|
"computed_at",
|
|
sa.DateTime(timezone=True),
|
|
server_default=sa.text("now()"),
|
|
nullable=False,
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["category_id"],
|
|
["category.id"],
|
|
name=op.f("fk_metric_spending_by_category_category_id_category"),
|
|
ondelete="CASCADE",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["root_category_id"],
|
|
["category.id"],
|
|
name=op.f("fk_metric_spending_by_category_root_category_id_category"),
|
|
ondelete="CASCADE",
|
|
),
|
|
sa.PrimaryKeyConstraint("id", name=op.f("pk_metric_spending_by_category")),
|
|
sa.UniqueConstraint(
|
|
"month",
|
|
"category_id",
|
|
name=op.f("uq_metric_spending_by_category_month_category_id"),
|
|
postgresql_nulls_not_distinct=True,
|
|
),
|
|
)
|
|
op.create_index(
|
|
op.f("ix_metric_spending_by_category_month"),
|
|
"metric_spending_by_category",
|
|
["month"],
|
|
unique=False,
|
|
)
|
|
op.create_table(
|
|
"cash_txn_tag",
|
|
sa.Column("txn_id", sa.Integer(), nullable=False),
|
|
sa.Column("ord", sa.Integer(), nullable=False),
|
|
sa.Column("category_id", sa.Integer(), nullable=False),
|
|
sa.ForeignKeyConstraint(
|
|
["category_id"],
|
|
["category.id"],
|
|
name=op.f("fk_cash_txn_tag_category_id_category"),
|
|
ondelete="CASCADE",
|
|
),
|
|
sa.ForeignKeyConstraint(
|
|
["txn_id"],
|
|
["cash_txn.id"],
|
|
name=op.f("fk_cash_txn_tag_txn_id_cash_txn"),
|
|
ondelete="CASCADE",
|
|
),
|
|
sa.PrimaryKeyConstraint("txn_id", "ord", name=op.f("pk_cash_txn_tag")),
|
|
)
|
|
op.create_index(
|
|
op.f("ix_cash_txn_tag_category_id"), "cash_txn_tag", ["category_id"], unique=False
|
|
)
|
|
# ### end Alembic commands ###
|
|
|
|
|
|
def downgrade() -> None:
|
|
# ### commands auto generated by Alembic - please adjust! ###
|
|
op.drop_index(op.f("ix_cash_txn_tag_category_id"), table_name="cash_txn_tag")
|
|
op.drop_table("cash_txn_tag")
|
|
op.drop_index(
|
|
op.f("ix_metric_spending_by_category_month"), table_name="metric_spending_by_category"
|
|
)
|
|
op.drop_table("metric_spending_by_category")
|
|
op.drop_index(op.f("ix_cash_txn_outcome_account_id"), table_name="cash_txn")
|
|
op.drop_index(op.f("ix_cash_txn_income_account_id"), table_name="cash_txn")
|
|
op.drop_index(op.f("ix_cash_txn_date"), table_name="cash_txn")
|
|
op.drop_table("cash_txn")
|
|
op.drop_table("trip")
|
|
op.drop_table("rule")
|
|
op.drop_table("raw_zenmoney_entity")
|
|
op.drop_table("raw_zenmoney_deletion")
|
|
op.drop_table("raw_cbr_rate")
|
|
op.drop_table("metric_runway")
|
|
op.drop_table("metric_refresh_log")
|
|
op.drop_table("metric_net_worth_daily")
|
|
op.drop_index(op.f("ix_metric_data_quality_check_name"), table_name="metric_data_quality")
|
|
op.drop_table("metric_data_quality")
|
|
op.drop_table("metric_cash_flow_monthly")
|
|
op.drop_table("merchant")
|
|
op.drop_table("fx_rate_daily")
|
|
op.drop_table("category")
|
|
# ### end Alembic commands ###
|
|
for enum_name in ("flow_type", "rule_kind", "rule_match_type"):
|
|
sa.Enum(name=enum_name).drop(op.get_bind(), checkfirst=True)
|