Dmitry
3b3ee4d682
POST /metrics/refresh ставит задачу в sync_job (source=METRICS_JOB) и отвечает 202, пересчёт делает worker; GET /metrics/status отдаёт refreshing и consistent. metric_refresh_log хранит failed_step и step_timings. Источники с needs="tinvest_token" не попадают в расписание без токена, tinvest/moex добавлены в default_schedule. Воркер трогает heartbeat-файл для healthcheck.
90 lines
3.0 KiB
Python
90 lines
3.0 KiB
Python
"""Sync bookkeeping: cursors, run log, manual job queue, per-source credentials."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import enum
|
|
import uuid
|
|
from datetime import datetime
|
|
from typing import Any
|
|
|
|
from sqlalchemy import ForeignKey, String, Text, func
|
|
from sqlalchemy.orm import Mapped, mapped_column
|
|
|
|
from fintracker.db.base import Base, db_enum
|
|
|
|
|
|
class RunStatus(enum.StrEnum):
|
|
running = "running"
|
|
ok = "ok"
|
|
error = "error"
|
|
|
|
|
|
METRICS_JOB = "metrics"
|
|
"""`sync_job.source` of a queued metrics rebuild. Not a registered source: the worker
|
|
routes it to `refresh_all` instead of `run_source`, and `POST /sync/{source}` answers 404."""
|
|
|
|
|
|
class JobStatus(enum.StrEnum):
|
|
queued = "queued"
|
|
running = "running"
|
|
done = "done"
|
|
error = "error"
|
|
|
|
|
|
class SyncState(Base):
|
|
"""One row per source: where the incremental sync left off."""
|
|
|
|
__tablename__ = "sync_state"
|
|
|
|
source: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
cursor: Mapped[str | None] = mapped_column(Text)
|
|
last_run_at: Mapped[datetime | None]
|
|
last_success_at: Mapped[datetime | None]
|
|
|
|
|
|
class SyncRun(Base):
|
|
__tablename__ = "sync_run"
|
|
|
|
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
|
source: Mapped[str] = mapped_column(String(64), index=True)
|
|
status: Mapped[RunStatus] = mapped_column(db_enum(RunStatus, "run_status"))
|
|
triggered_by: Mapped[str] = mapped_column(String(32))
|
|
"""schedule | manual | cli"""
|
|
started_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
|
finished_at: Mapped[datetime | None]
|
|
cursor_before: Mapped[str | None] = mapped_column(Text)
|
|
cursor_after: Mapped[str | None] = mapped_column(Text)
|
|
counts: Mapped[dict[str, Any] | None]
|
|
warnings: Mapped[list[Any] | None]
|
|
error: Mapped[str | None] = mapped_column(Text)
|
|
|
|
|
|
class SyncJob(Base):
|
|
"""Manual trigger queue: API inserts, worker picks up (plan §2.4). Also carries the metrics
|
|
rebuild requested by `POST /metrics/refresh`, as `source == METRICS_JOB`."""
|
|
|
|
__tablename__ = "sync_job"
|
|
|
|
id: Mapped[uuid.UUID] = mapped_column(primary_key=True, default=uuid.uuid4)
|
|
source: Mapped[str] = mapped_column(String(64), index=True)
|
|
status: Mapped[JobStatus] = mapped_column(db_enum(JobStatus, "job_status"), index=True)
|
|
requested_at: Mapped[datetime] = mapped_column(server_default=func.now())
|
|
started_at: Mapped[datetime | None]
|
|
finished_at: Mapped[datetime | None]
|
|
run_id: Mapped[uuid.UUID | None] = mapped_column(ForeignKey("sync_run.id", ondelete="SET NULL"))
|
|
error: Mapped[str | None] = mapped_column(Text)
|
|
|
|
|
|
class SourceCredential(Base):
|
|
"""Rotating credentials a source manages itself (e.g. ZenMoney refresh token).
|
|
|
|
Static tokens still come from the environment; this is for what the worker must
|
|
be able to update unattended.
|
|
"""
|
|
|
|
__tablename__ = "source_credential"
|
|
|
|
source: Mapped[str] = mapped_column(String(64), primary_key=True)
|
|
payload: Mapped[dict[str, Any]]
|
|
updated_at: Mapped[datetime] = mapped_column(server_default=func.now(), onupdate=func.now())
|