Dmitry
c55fe19e48
Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
96 lines
3.7 KiB
Python
96 lines
3.7 KiB
Python
"""Run one source sync end to end: lock, run log, cursor, error capture."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import logging
|
|
import traceback
|
|
from datetime import UTC, datetime
|
|
|
|
from sqlalchemy import select
|
|
|
|
from fintracker.config import get_settings
|
|
from fintracker.db import get_engine, get_sessionmaker
|
|
from fintracker.metrics.refresh import refresh_all
|
|
from fintracker.models import RunStatus, SyncRun, SyncState
|
|
from fintracker.sources import registry
|
|
from fintracker.sources.base import SyncContext, SyncResult
|
|
from fintracker.worker.locks import try_advisory_lock
|
|
|
|
log = logging.getLogger(__name__)
|
|
|
|
|
|
class Skipped(Exception):
|
|
"""Another run of the same source holds the lock."""
|
|
|
|
|
|
async def run_source(name: str, triggered_by: str) -> SyncRun:
|
|
source = registry.get(name)
|
|
settings = get_settings()
|
|
# the lock lives on its own connection for the whole run
|
|
async with (
|
|
get_engine().connect() as lock_conn,
|
|
try_advisory_lock(lock_conn, f"sync:{name}") as got,
|
|
):
|
|
if not got:
|
|
raise Skipped(name)
|
|
async with get_sessionmaker()() as session:
|
|
state = await session.get(SyncState, name)
|
|
if state is None:
|
|
state = SyncState(source=name)
|
|
session.add(state)
|
|
run = SyncRun(
|
|
source=name,
|
|
status=RunStatus.running,
|
|
triggered_by=triggered_by,
|
|
cursor_before=state.cursor,
|
|
)
|
|
session.add(run)
|
|
state.last_run_at = datetime.now(UTC)
|
|
await session.commit()
|
|
run_id = run.id
|
|
|
|
ctx = SyncContext(
|
|
session=session,
|
|
settings=settings,
|
|
cursor_before=state.cursor,
|
|
triggered_by=triggered_by,
|
|
)
|
|
try:
|
|
result: SyncResult = await source.sync(ctx)
|
|
except Exception as exc:
|
|
await session.rollback()
|
|
log.exception("sync %s failed", name)
|
|
async with get_sessionmaker()() as s2:
|
|
run2 = await s2.get(SyncRun, run_id)
|
|
assert run2 is not None
|
|
run2.status = RunStatus.error
|
|
run2.finished_at = datetime.now(UTC)
|
|
run2.error = f"{type(exc).__name__}: {exc}\n{traceback.format_exc()[-4000:]}"
|
|
await s2.commit()
|
|
return run2
|
|
|
|
run = (await session.execute(select(SyncRun).where(SyncRun.id == run_id))).scalar_one()
|
|
state = await session.get(SyncState, name)
|
|
assert state is not None
|
|
if result.cursor_after is not None:
|
|
state.cursor = result.cursor_after
|
|
state.last_success_at = datetime.now(UTC)
|
|
run.status = RunStatus.ok
|
|
run.finished_at = datetime.now(UTC)
|
|
run.cursor_after = state.cursor
|
|
run.counts = dict(result.counts)
|
|
run.warnings = list(result.warnings)
|
|
await session.commit()
|
|
log.info("sync %s ok: %s", name, result.counts)
|
|
if result.changed:
|
|
entry = await refresh_all(session, trigger=f"sync:{name}")
|
|
if entry.error:
|
|
# the download itself succeeded, so the cursor stays advanced (a replay
|
|
# would fix nothing); the run is still an error, because the metrics the
|
|
# UI reads are now stale. `scheduler._finish_job` picks this up.
|
|
run.status = RunStatus.error
|
|
run.error = f"metrics refresh failed: {entry.error}"
|
|
await session.commit()
|
|
log.error("sync %s: metrics refresh failed", name)
|
|
return run
|