feat(worker): ручной пересчёт метрик через очередь, диагностика шагов и heartbeat

POST /metrics/refresh ставит задачу в sync_job (source=METRICS_JOB) и отвечает 202, пересчёт делает worker; GET /metrics/status отдаёт refreshing и consistent. metric_refresh_log хранит failed_step и step_timings. Источники с needs="tinvest_token" не попадают в расписание без токена, tinvest/moex добавлены в default_schedule. Воркер трогает heartbeat-файл для healthcheck.

backend/tests/api/test_metrics_api.py

@@ -1,8 +1,19 @@
 from datetime import timedelta
 
+from sqlalchemy import select
+
 from factories import make_account, make_category, make_txn, month_back
 from fintracker.analytics import today_local
-from fintracker.models import AccountRole
+from fintracker.models import METRICS_JOB, AccountRole, JobStatus, SyncJob
+from fintracker.worker.scheduler import _process_queued_jobs
+
+
+async def _queue_and_run(client, auth_headers):
+    """POST /metrics/refresh, then let the worker take the job, as it does every 5 s."""
+    r = await client.post("/api/v1/metrics/refresh", headers=auth_headers)
+    assert r.status_code == 202, r.text
+    await _process_queued_jobs()
+    return r.json()
 
 
 async def test_refresh_populates_every_metric_endpoint(client, auth_headers):
@@ -20,15 +31,20 @@ async def test_refresh_populates_every_metric_endpoint(client, auth_headers):
     )
     await make_txn(today_local() - timedelta(days=1), outcome="500", outcome_account_id=card)
 
-    assert (await client.get("/api/v1/metrics/status", headers=auth_headers)).json() is None
+    before = (await client.get("/api/v1/metrics/status", headers=auth_headers)).json()
+    assert before == {"last_refresh": None, "consistent": True, "refreshing": False}
 
-    r = await client.post("/api/v1/metrics/refresh", headers=auth_headers)
-    assert r.status_code == 202
-    assert r.json()["error"] is None
-    assert r.json()["finished_at"] is not None
+    job = await _queue_and_run(client, auth_headers)
+    assert job["status"] == "queued"
 
     status = (await client.get("/api/v1/metrics/status", headers=auth_headers)).json()
-    assert status["trigger"] == "manual"
+    assert status["last_refresh"]["trigger"] == "manual"
+    assert status["last_refresh"]["error"] is None
+    assert status["last_refresh"]["finished_at"] is not None
+    assert status["last_refresh"]["failed_step"] is None
+    assert status["last_refresh"]["step_timings"]  # every step is timed
+    assert status["consistent"] is True
+    assert status["refreshing"] is False
 
     series = (await client.get("/api/v1/networth/series", headers=auth_headers)).json()
     assert series
@@ -81,7 +97,7 @@ async def test_spending_rejects_a_bad_month(client, auth_headers):
 
 
 async def test_empty_database_reports_no_transactions(client, auth_headers):
-    await client.post("/api/v1/metrics/refresh", headers=auth_headers)
+    await _queue_and_run(client, auth_headers)
     quality = (await client.get("/api/v1/data-quality", headers=auth_headers)).json()
     assert [row["check_name"] for row in quality] == ["no_transactions"]
     assert (await client.get("/api/v1/runway", headers=auth_headers)).json()[
@@ -89,3 +105,74 @@ async def test_empty_database_reports_no_transactions(client, auth_headers):
     ] is None
     assert (await client.get("/api/v1/networth/series", headers=auth_headers)).json() == []
     assert (await client.get("/api/v1/spending/categories", headers=auth_headers)).json() == []
+
+
+async def test_refresh_returns_before_the_work_is_done(client, auth_headers):
+    """The 202 is honest: nothing has been rebuilt until the worker picks the job up."""
+    r = await client.post("/api/v1/metrics/refresh", headers=auth_headers)
+    assert r.status_code == 202
+    assert r.json()["source"] == METRICS_JOB
+
+    status = (await client.get("/api/v1/metrics/status", headers=auth_headers)).json()
+    assert status["last_refresh"] is None
+    assert status["refreshing"] is True
+
+    await _process_queued_jobs()
+
+    status = (await client.get("/api/v1/metrics/status", headers=auth_headers)).json()
+    assert status["refreshing"] is False
+    assert status["last_refresh"] is not None
+
+
+async def test_a_waiting_refresh_is_shared_not_duplicated(client, auth_headers):
+    first = (await client.post("/api/v1/metrics/refresh", headers=auth_headers)).json()
+    second = (await client.post("/api/v1/metrics/refresh", headers=auth_headers)).json()
+    assert first["id"] == second["id"]
+
+    from fintracker.db import get_sessionmaker
+
+    async with get_sessionmaker()() as session:
+        jobs = (
+            (await session.execute(select(SyncJob).where(SyncJob.source == METRICS_JOB)))
+            .scalars()
+            .all()
+        )
+    assert len(jobs) == 1
+
+
+async def test_a_refresh_that_fails_part_way_is_reported_as_inconsistent(
+    client, auth_headers, monkeypatch
+):
+    from fintracker.db import get_sessionmaker
+    from fintracker.metrics import refresh as refresh_module
+
+    async def fine(session):
+        pass
+
+    async def boom(session):
+        raise RuntimeError("step blew up")
+
+    monkeypatch.setattr("fintracker.analytics.register_steps", lambda: None)
+    monkeypatch.setattr(refresh_module, "STEPS", [("fine", fine), ("boom", boom), ("never", fine)])
+
+    await _queue_and_run(client, auth_headers)
+
+    status = (await client.get("/api/v1/metrics/status", headers=auth_headers)).json()
+    last = status["last_refresh"]
+    assert last["failed_step"] == "boom"
+    assert "step blew up" in last["error"]
+    assert list(last["step_timings"]) == ["fine"]  # 'never' did not run, 'boom' did not finish
+    assert status["consistent"] is False
+
+    async with get_sessionmaker()() as session:
+        job = (
+            await session.execute(select(SyncJob).where(SyncJob.source == METRICS_JOB))
+        ).scalar_one()
+    assert job.status == JobStatus.error
+
+    # a later clean run makes the tables one snapshot again
+    monkeypatch.setattr(refresh_module, "STEPS", [("fine", fine)])
+    await _queue_and_run(client, auth_headers)
+    status = (await client.get("/api/v1/metrics/status", headers=auth_headers)).json()
+    assert status["consistent"] is True
+    assert status["last_refresh"]["failed_step"] is None

backend/tests/test_schedule.py

@@ -0,0 +1,49 @@
+"""The timetable only names sources that exist, and leaves off the ones that cannot run."""
+
+from __future__ import annotations
+
+from fintracker.config import Settings
+from fintracker.sources import registry
+from fintracker.worker.jobs import default_schedule, scheduled
+
+
+def test_every_scheduled_source_is_registered_once():
+    names = [s.source for s in default_schedule()]
+    assert len(names) == len(set(names))
+    assert set(names) <= set(registry.names())
+
+
+def test_every_live_source_is_on_the_timetable():
+    assert {s.source for s in default_schedule()} == set(registry.names())
+
+
+def test_sources_needing_a_token_are_left_off_without_one():
+    ready, skipped = scheduled(Settings.model_construct(tinvest_token=None))
+    assert {s.source for s in skipped} == {"tinvest", "tinvest_events"}
+    assert not {s.source for s in ready} & {"tinvest", "tinvest_events"}
+
+    ready, skipped = scheduled(Settings.model_construct(tinvest_token="t.x"))
+    assert skipped == []
+    assert {"tinvest", "moex", "tinvest_events", "moex_payouts"} <= {s.source for s in ready}
+
+
+def test_needs_names_a_real_setting():
+    for spec in default_schedule():
+        if spec.needs is not None:
+            assert spec.needs in Settings.model_fields
+
+
+def test_worker_timetable_has_the_heartbeat_and_the_queue_poller():
+    from fintracker.worker.scheduler import build_scheduler
+
+    ids = {j.id for j in build_scheduler().get_jobs()}
+    assert {"heartbeat", "poll-sync-jobs", "sync:zenmoney", "sync:moex", "sync:moex_payouts"} <= ids
+
+
+async def test_heartbeat_touches_its_file(tmp_path, monkeypatch):
+    from fintracker.worker import scheduler
+
+    beat = tmp_path / "beat"
+    monkeypatch.setattr(scheduler, "HEARTBEAT_FILE", beat)
+    await scheduler._heartbeat()
+    assert beat.exists()