feat(ledger): импорт отчётов в леджер — приём, дедупликация, pending_instrument
Поток: upload -> raw_report_file (sha256 UNIQUE) -> parse -> raw_report_line,
событий в леджере ещё нет -> preview -> POST /imports/{id}/commit ->
ledger/ingest.py резолвит инструмент, считает dedupe_key, пишет event
confirmed | shadow (plan §1.6 B) — сравнивая account.primary_event_source
с источником отчёта, а не гадая. Нерезолвленный инструмент ждёт в
pending_instrument, никогда не угадывается; POST /instruments/pending/{id}/resolve
привязывает и пересобирает лоты.
ledger/dedupe.py — shadow-матчинг случая B двумя проходами (точная дата, затем
±1 рабочий день, жадно 1:1, |price| ±0,5 %). Шаги shadow_dedupe и
report_reconcile зарегистрированы перед quality: оба говорят через FINDINGS.
/instruments/pending регистрируется в app.py ДО routers/instruments.py:
FastAPI сопоставляет маршруты по порядку, и /instruments/{id} с типом int
отвечает 422 на нечисловой сегмент, а не проваливается дальше.
Контракт — docs/ai/import-contract.md, общий для бэкенда и Flutter.
This commit is contained in:
Dmitry
@@ -0,0 +1,168 @@
|
||||
"""импорт брокерских отчётов: raw_report_file, raw_report_line, pending_instrument
|
||||
|
||||
Revision ID: ee170b3e7872
|
||||
Revises: 3e2977b9e577
|
||||
Create Date: 2026-09-18 15:35:07.287919
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Sequence
|
||||
|
||||
import sqlalchemy as sa
|
||||
from alembic import op
|
||||
from sqlalchemy.dialects import postgresql
|
||||
|
||||
revision: str = "ee170b3e7872"
|
||||
down_revision: str | None = "3e2977b9e577"
|
||||
branch_labels: str | Sequence[str] | None = None
|
||||
depends_on: str | Sequence[str] | None = None
|
||||
|
||||
|
||||
def upgrade() -> None:
|
||||
# Autogenerate cannot see a new member of an existing Postgres enum — reports are a
|
||||
# primary_event_source too, and a universal CSV export is one of them.
|
||||
op.execute("ALTER TYPE event_source ADD VALUE IF NOT EXISTS 'csv' BEFORE 'manual'")
|
||||
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.create_table(
|
||||
"raw_report_file",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("broker", sa.String(length=32), nullable=False),
|
||||
sa.Column("filename", sa.String(length=512), nullable=False),
|
||||
sa.Column("sha256", sa.String(length=64), nullable=False),
|
||||
sa.Column("size_bytes", sa.Integer(), nullable=False),
|
||||
sa.Column("content", sa.LargeBinary(), nullable=True),
|
||||
sa.Column("account_id", sa.Integer(), nullable=True),
|
||||
sa.Column("account_external_id", sa.String(length=128), nullable=True),
|
||||
sa.Column("period_from", sa.Date(), nullable=True),
|
||||
sa.Column("period_to", sa.Date(), nullable=True),
|
||||
sa.Column("parser_name", sa.String(length=32), nullable=True),
|
||||
sa.Column("parser_version", sa.String(length=32), nullable=True),
|
||||
sa.Column(
|
||||
"parse_status",
|
||||
sa.Enum("uploaded", "parsed", "committed", "failed", name="report_parse_status"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("error", sa.Text(), nullable=True),
|
||||
sa.Column("warnings", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.Column("counts", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.Column("uploaded_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("committed_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["account_id"],
|
||||
["account.id"],
|
||||
name=op.f("fk_raw_report_file_account_id_account"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_raw_report_file")),
|
||||
)
|
||||
op.create_index("ix_raw_report_file_account", "raw_report_file", ["account_id"], unique=False)
|
||||
op.create_index("uq_raw_report_file_sha256", "raw_report_file", ["sha256"], unique=True)
|
||||
op.create_table(
|
||||
"pending_instrument",
|
||||
sa.Column("id", sa.Integer(), nullable=False),
|
||||
sa.Column("source", sa.String(length=32), nullable=False),
|
||||
sa.Column("source_key", sa.String(length=256), nullable=False),
|
||||
sa.Column("isin", sa.String(length=12), nullable=True),
|
||||
sa.Column("ticker", sa.String(length=32), nullable=True),
|
||||
sa.Column("board", sa.String(length=16), nullable=True),
|
||||
sa.Column("name", sa.String(length=256), nullable=True),
|
||||
sa.Column("currency", sa.String(length=3), nullable=True),
|
||||
sa.Column("asset_class_hint", sa.String(length=16), nullable=True),
|
||||
sa.Column("first_seen_file_id", sa.Integer(), nullable=True),
|
||||
sa.Column("occurrences", sa.BigInteger(), nullable=False),
|
||||
sa.Column("sample_quantity", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column("sample_price", sa.Numeric(precision=24, scale=10), nullable=True),
|
||||
sa.Column(
|
||||
"status",
|
||||
sa.Enum("pending", "resolved", "ignored", name="pending_instrument_status"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column("instrument_id", sa.Integer(), nullable=True),
|
||||
sa.Column("resolved_at", sa.DateTime(timezone=True), nullable=True),
|
||||
sa.Column("meta", postgresql.JSONB(astext_type=sa.Text()), nullable=True),
|
||||
sa.Column(
|
||||
"created_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.Column(
|
||||
"updated_at",
|
||||
sa.DateTime(timezone=True),
|
||||
server_default=sa.text("now()"),
|
||||
nullable=False,
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["first_seen_file_id"],
|
||||
["raw_report_file.id"],
|
||||
name=op.f("fk_pending_instrument_first_seen_file_id_raw_report_file"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["instrument_id"],
|
||||
["instrument.id"],
|
||||
name=op.f("fk_pending_instrument_instrument_id_instrument"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("id", name=op.f("pk_pending_instrument")),
|
||||
sa.UniqueConstraint(
|
||||
"source", "source_key", name=op.f("uq_pending_instrument_source_source_key")
|
||||
),
|
||||
)
|
||||
op.create_table(
|
||||
"raw_report_line",
|
||||
sa.Column("file_id", sa.Integer(), nullable=False),
|
||||
sa.Column("line_no", sa.Integer(), nullable=False),
|
||||
sa.Column("section", sa.String(length=64), nullable=True),
|
||||
sa.Column("payload", postgresql.JSONB(astext_type=sa.Text()), nullable=False),
|
||||
sa.Column("dedupe_key", sa.String(length=256), nullable=True),
|
||||
sa.Column("event_id", sa.Integer(), nullable=True),
|
||||
sa.ForeignKeyConstraint(
|
||||
["event_id"],
|
||||
["event.id"],
|
||||
name=op.f("fk_raw_report_line_event_id_event"),
|
||||
ondelete="SET NULL",
|
||||
),
|
||||
sa.ForeignKeyConstraint(
|
||||
["file_id"],
|
||||
["raw_report_file.id"],
|
||||
name=op.f("fk_raw_report_line_file_id_raw_report_file"),
|
||||
ondelete="CASCADE",
|
||||
),
|
||||
sa.PrimaryKeyConstraint("file_id", "line_no", name=op.f("pk_raw_report_line")),
|
||||
)
|
||||
op.create_index(
|
||||
op.f("ix_raw_report_line_dedupe_key"), "raw_report_line", ["dedupe_key"], unique=False
|
||||
)
|
||||
# ### end Alembic commands ###
|
||||
|
||||
|
||||
def downgrade() -> None:
|
||||
# ### commands auto generated by Alembic - please adjust! ###
|
||||
op.drop_index(op.f("ix_raw_report_line_dedupe_key"), table_name="raw_report_line")
|
||||
op.drop_table("raw_report_line")
|
||||
op.drop_table("pending_instrument")
|
||||
op.drop_index("uq_raw_report_file_sha256", table_name="raw_report_file")
|
||||
op.drop_index("ix_raw_report_file_account", table_name="raw_report_file")
|
||||
op.drop_table("raw_report_file")
|
||||
# ### end Alembic commands ###
|
||||
|
||||
# Dropping a table leaves its enum type behind, and a re-upgrade would then fail.
|
||||
# `event_source` keeps its extra member: Postgres cannot remove one, and an unused
|
||||
# value is harmless.
|
||||
sa.Enum(name="pending_instrument_status").drop(op.get_bind(), checkfirst=True)
|
||||
sa.Enum(name="report_parse_status").drop(op.get_bind(), checkfirst=True)
|
||||
Reference in New Issue
Block a user