feat(worker): ручной пересчёт метрик через очередь, диагностика шагов и heartbeat

POST /metrics/refresh ставит задачу в sync_job (source=METRICS_JOB) и отвечает 202, пересчёт делает worker; GET /metrics/status отдаёт refreshing и consistent. metric_refresh_log хранит failed_step и step_timings. Источники с needs="tinvest_token" не попадают в расписание без токена, tinvest/moex добавлены в default_schedule. Воркер трогает heartbeat-файл для healthcheck.

backend/src/fintracker/worker/jobs.py

@@ -9,6 +9,8 @@ from apscheduler.triggers.base import BaseTrigger
 from apscheduler.triggers.combining import OrTrigger
 from apscheduler.triggers.cron import CronTrigger
 
+from fintracker.config import Settings
+
 
 @dataclass(frozen=True)
 class JobSpec:
@@ -16,10 +18,13 @@ class JobSpec:
     trigger: BaseTrigger
     """A source appears at most once: `worker/scheduler.py` keys jobs by `sync:<source>`,
     so several times of day are one `OrTrigger`, not several specs."""
+    needs: str | None = None
+    """Name of a `Settings` field that must be set for the source to run at all. Without it
+    the job is left off the timetable: a run would only fail, hourly, and bury real errors."""
 
 
 MSK = ZoneInfo("Europe/Moscow")
-"""CBR publishes on Moscow time, so those jobs pin the zone instead of following settings."""
+"""CBR and MOEX publish on Moscow time, so those jobs pin the zone instead of following settings."""
 
 
 def default_schedule() -> list[JobSpec]:
@@ -36,4 +41,24 @@ def default_schedule() -> list[JobSpec]:
                 ]
             ),
         ),
+        # Broker operations and balances, every three hours. The MOEX run below is placed
+        # after a T-Invest one, because it prices whatever the ledger says is held.
+        JobSpec(
+            "tinvest", CronTrigger(hour="8-23/3", minute=10, timezone=MSK), needs="tinvest_token"
+        ),
+        # Prices: after the open, mid-session, after the close (main session ends 18:50 MSK),
+        # and once more for the evening session.
+        JobSpec("moex", CronTrigger(hour="10,14,19,23", minute=20, timezone=MSK)),
+        # Announced dividends and coupons change rarely: once a day, before the market opens.
+        JobSpec(
+            "tinvest_events", CronTrigger(hour=6, minute=30, timezone=MSK), needs="tinvest_token"
+        ),
+        JobSpec("moex_payouts", CronTrigger(hour=6, minute=45, timezone=MSK)),
     ]
+
+
+def scheduled(settings: Settings) -> tuple[list[JobSpec], list[JobSpec]]:
+    """(to run, left off because a required setting is missing)."""
+    specs = default_schedule()
+    ready = [s for s in specs if s.needs is None or getattr(settings, s.needs)]
+    return ready, [s for s in specs if s not in ready]

backend/src/fintracker/worker/scheduler.py

@@ -6,20 +6,30 @@ import asyncio
 import logging
 import signal
 from datetime import UTC, datetime
+from pathlib import Path
 
 from apscheduler.schedulers.asyncio import AsyncIOScheduler
 from sqlalchemy import select
 
 from fintracker.config import get_settings
 from fintracker.db import get_sessionmaker, reset_engine
-from fintracker.models import JobStatus, SyncJob
+from fintracker.metrics.refresh import refresh_all
+from fintracker.models import METRICS_JOB, JobStatus, SyncJob
 from fintracker.sources import registry
-from fintracker.worker.jobs import default_schedule
+from fintracker.worker.jobs import scheduled
 from fintracker.worker.runner import Skipped, run_source
 
 log = logging.getLogger(__name__)
 
 POLL_SECONDS = 5
+HEARTBEAT_SECONDS = 10
+HEARTBEAT_FILE = Path("/tmp/fintracker-worker-heartbeat")
+"""Touched by its own job, so a long sync (which blocks `_process_queued_jobs`, not the event
+loop) does not look like a dead worker. The compose healthcheck reads its mtime."""
+
+
+async def _heartbeat() -> None:
+    HEARTBEAT_FILE.touch()
 
 
 async def _scheduled(source: str) -> None:
@@ -43,6 +53,9 @@ async def _process_queued_jobs() -> None:
             .all()
         )
     for job in jobs:
+        if job.source == METRICS_JOB:
+            await _run_metrics_job(job.id)
+            continue
         if job.source not in registry.names():
             await _finish_job(job.id, JobStatus.error, error=f"unknown source {job.source}")
             continue
@@ -56,6 +69,20 @@ async def _process_queued_jobs() -> None:
         await _finish_job(job.id, status, run_id=run.id, error=run.error)
 
 
+async def _run_metrics_job(job_id) -> None:
+    await _mark_running(job_id)
+    try:
+        async with get_sessionmaker()() as session:
+            entry = await refresh_all(session, trigger="manual")
+    except Exception as exc:  # the lock connection or the log insert, not a step (those are caught)
+        log.exception("metrics job failed")
+        await _finish_job(job_id, JobStatus.error, error=f"{type(exc).__name__}: {exc}")
+        return
+    await _finish_job(
+        job_id, JobStatus.done if entry.error is None else JobStatus.error, error=entry.error
+    )
+
+
 async def _mark_running(job_id) -> None:
     async with get_sessionmaker()() as session:
         job = await session.get(SyncJob, job_id)
@@ -79,7 +106,10 @@ async def _finish_job(job_id, status: JobStatus, *, run_id=None, error: str | No
 def build_scheduler() -> AsyncIOScheduler:
     settings = get_settings()
     scheduler = AsyncIOScheduler(timezone=settings.timezone)
-    for spec in default_schedule():
+    specs, skipped = scheduled(settings)
+    for spec in skipped:
+        log.warning("not scheduling %s: %s is not set", spec.source, spec.needs)
+    for spec in specs:
         scheduler.add_job(
             _scheduled,
             spec.trigger,
@@ -89,6 +119,15 @@ def build_scheduler() -> AsyncIOScheduler:
             coalesce=True,
             misfire_grace_time=600,
         )
+    scheduler.add_job(
+        _heartbeat,
+        "interval",
+        seconds=HEARTBEAT_SECONDS,
+        id="heartbeat",
+        max_instances=1,
+        coalesce=True,
+        next_run_time=datetime.now(UTC),
+    )
     scheduler.add_job(
         _process_queued_jobs,
         "interval",