feat(sources): контракт источников, worker и синк ZenMoney + ЦБ
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
"""Run one source sync end to end: lock, run log, cursor, error capture."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import traceback
|
||||
from datetime import UTC, datetime
|
||||
|
||||
from sqlalchemy import select
|
||||
|
||||
from fintracker.config import get_settings
|
||||
from fintracker.db import get_engine, get_sessionmaker
|
||||
from fintracker.metrics.refresh import refresh_all
|
||||
from fintracker.models import RunStatus, SyncRun, SyncState
|
||||
from fintracker.sources import registry
|
||||
from fintracker.sources.base import SyncContext, SyncResult
|
||||
from fintracker.worker.locks import try_advisory_lock
|
||||
|
||||
log = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class Skipped(Exception):
|
||||
"""Another run of the same source holds the lock."""
|
||||
|
||||
|
||||
async def run_source(name: str, triggered_by: str) -> SyncRun:
|
||||
source = registry.get(name)
|
||||
settings = get_settings()
|
||||
# the lock lives on its own connection for the whole run
|
||||
async with (
|
||||
get_engine().connect() as lock_conn,
|
||||
try_advisory_lock(lock_conn, f"sync:{name}") as got,
|
||||
):
|
||||
if not got:
|
||||
raise Skipped(name)
|
||||
async with get_sessionmaker()() as session:
|
||||
state = await session.get(SyncState, name)
|
||||
if state is None:
|
||||
state = SyncState(source=name)
|
||||
session.add(state)
|
||||
run = SyncRun(
|
||||
source=name,
|
||||
status=RunStatus.running,
|
||||
triggered_by=triggered_by,
|
||||
cursor_before=state.cursor,
|
||||
)
|
||||
session.add(run)
|
||||
state.last_run_at = datetime.now(UTC)
|
||||
await session.commit()
|
||||
run_id = run.id
|
||||
|
||||
ctx = SyncContext(
|
||||
session=session,
|
||||
settings=settings,
|
||||
cursor_before=state.cursor,
|
||||
triggered_by=triggered_by,
|
||||
)
|
||||
try:
|
||||
result: SyncResult = await source.sync(ctx)
|
||||
except Exception as exc:
|
||||
await session.rollback()
|
||||
log.exception("sync %s failed", name)
|
||||
async with get_sessionmaker()() as s2:
|
||||
run2 = await s2.get(SyncRun, run_id)
|
||||
assert run2 is not None
|
||||
run2.status = RunStatus.error
|
||||
run2.finished_at = datetime.now(UTC)
|
||||
run2.error = f"{type(exc).__name__}: {exc}\n{traceback.format_exc()[-4000:]}"
|
||||
await s2.commit()
|
||||
return run2
|
||||
|
||||
run = (await session.execute(select(SyncRun).where(SyncRun.id == run_id))).scalar_one()
|
||||
state = await session.get(SyncState, name)
|
||||
assert state is not None
|
||||
if result.cursor_after is not None:
|
||||
state.cursor = result.cursor_after
|
||||
state.last_success_at = datetime.now(UTC)
|
||||
run.status = RunStatus.ok
|
||||
run.finished_at = datetime.now(UTC)
|
||||
run.cursor_after = state.cursor
|
||||
run.counts = dict(result.counts)
|
||||
run.warnings = list(result.warnings)
|
||||
await session.commit()
|
||||
log.info("sync %s ok: %s", name, result.counts)
|
||||
if result.changed:
|
||||
entry = await refresh_all(session, trigger=f"sync:{name}")
|
||||
if entry.error:
|
||||
# the download itself succeeded, so the cursor stays advanced (a replay
|
||||
# would fix nothing); the run is still an error, because the metrics the
|
||||
# UI reads are now stale. `scheduler._finish_job` picks this up.
|
||||
run.status = RunStatus.error
|
||||
run.error = f"metrics refresh failed: {entry.error}"
|
||||
await session.commit()
|
||||
log.error("sync %s: metrics refresh failed", name)
|
||||
return run
|
||||
Reference in New Issue
Block a user