feat(sources): контракт источников, worker и синк ZenMoney + ЦБ

Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
2026-09-18 13:43:49 +03:00
parent 3fc7a954b9
commit c55fe19e48
31 changed files with 3713 additions and 0 deletions
@@ -0,0 +1,153 @@
+"""CBR source: which currencies are asked for, windows-1251 parsing, nominal handling."""
+
+from __future__ import annotations
+
+from datetime import date, timedelta
+from decimal import Decimal
+from itertools import pairwise
+
+import httpx
+from sqlalchemy import select
+
+from factories import make_account, make_txn
+from fintracker.analytics import today_local
+from fintracker.config import Settings
+from fintracker.db import get_sessionmaker
+from fintracker.models import RawCbrRate
+from fintracker.sources.cbr.client import DAILY_URL, DYNAMIC_URL, split_range
+from fintracker.sources.cbr.sync import CbrSource
+
+USD_ID = "R01235"
+JPY_ID = "R01820"
+
+
+def settings() -> Settings:
+    return Settings()
+
+
+async def stored_rates() -> dict[tuple[str, str], tuple[int, Decimal]]:
+    async with get_sessionmaker()() as session:
+        rows = (await session.execute(select(RawCbrRate))).scalars().all()
+        return {(r.ccy, r.rate_date.isoformat()): (r.nominal, r.value) for r in rows}
+
+
+def route_params(route) -> list[dict[str, str]]:
+    return [dict(call.request.url.params) for call in route.calls]
+
+
+def mock_cbr(mock_http, fixture_bytes, *, dynamic: dict[str, str] | None = None):
+    daily = mock_http.get(DAILY_URL).mock(
+        return_value=httpx.Response(200, content=fixture_bytes("cbr", "daily.xml"))
+    )
+    files = dynamic or {USD_ID: "dynamic_usd.xml", JPY_ID: "dynamic_jpy.xml"}
+
+    def _dynamic(request: httpx.Request) -> httpx.Response:
+        cbr_id = request.url.params.get("VAL_NM_RQ", "")
+        name = files.get(cbr_id)
+        if name is None:
+            return httpx.Response(200, content=b"<ValCurs></ValCurs>")
+        return httpx.Response(200, content=fixture_bytes("cbr", name))
+
+    return daily, mock_http.get(DYNAMIC_URL).mock(side_effect=_dynamic)
+
+
+async def test_only_currencies_in_use_are_requested(app, mock_http, fixture_bytes, run_sync):
+    await make_account(currency="RUB")
+    await make_account(currency="USD", source_id="usd-cash")
+    await make_txn(date(2026, 9, 1), outcome="100", outcome_currency="USD")
+
+    _, dynamic = mock_cbr(mock_http, fixture_bytes)
+    result = await run_sync(CbrSource(), settings=settings())
+
+    requested = {params["VAL_NM_RQ"] for params in route_params(dynamic)}
+    assert requested == {USD_ID}  # EUR and JPY are quoted by CBR but unused here
+    assert result.counts == {"currencies": 1, "rates": 3}
+    assert result.warnings == []
+    assert result.changed is True
+    assert result.cursor_after is not None
+
+
+async def test_values_and_nominal_are_stored_as_printed(app, mock_http, fixture_bytes, run_sync):
+    await make_account(currency="JPY", source_id="jpy")
+    await make_txn(date(2026, 9, 1), outcome="1000", outcome_currency="JPY")
+
+    mock_cbr(mock_http, fixture_bytes)
+    await run_sync(CbrSource(), settings=settings())
+
+    rates = await stored_rates()
+    # JPY is quoted per 100 units: both parts are kept, pricing/fx.py does the division
+    assert rates[("JPY", "2026-09-01")] == (100, Decimal("61.5432"))
+    assert rates[("JPY", "2026-09-02")] == (100, Decimal("62.0000"))
+
+
+async def test_currency_cbr_does_not_quote_is_a_warning(app, mock_http, fixture_bytes, run_sync):
+    await make_account(currency="USD", source_id="usd")
+    await make_account(currency="XAU", source_id="gold")
+    await make_txn(date(2026, 9, 1), outcome="1", outcome_currency="BTC")
+
+    _, dynamic = mock_cbr(mock_http, fixture_bytes)
+    result = await run_sync(CbrSource(), settings=settings())
+
+    assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID}
+    assert sorted(result.warnings) == [
+        "BTC is not quoted by CBR — skipped (crypto or metal?)",
+        "XAU is not quoted by CBR — skipped (crypto or metal?)",
+    ]
+    assert result.counts["currencies"] == 1
+    assert result.counts["rates"] == 3
+
+
+async def test_range_starts_before_the_first_transaction(app, mock_http, fixture_bytes, run_sync):
+    first = today_local() - timedelta(days=100)
+    await make_account(currency="USD", source_id="usd")
+    await make_txn(first, outcome="10", outcome_currency="USD")
+
+    _, dynamic = mock_cbr(mock_http, fixture_bytes)
+    await run_sync(CbrSource(), settings=settings())
+
+    params = route_params(dynamic)[0]
+    assert params["date_req1"] == (first - timedelta(days=7)).strftime("%d/%m/%Y")
+    assert params["date_req2"] == today_local().strftime("%d/%m/%Y")
+
+
+async def test_cursor_shortens_the_range_and_reruns_are_idempotent(
+    app, mock_http, fixture_bytes, run_sync
+):
+    await make_account(currency="USD", source_id="usd")
+    await make_txn(today_local() - timedelta(days=10), outcome="10", outcome_currency="USD")
+
+    _, dynamic = mock_cbr(mock_http, fixture_bytes)
+    first = await run_sync(CbrSource(), settings=settings())
+    before = await stored_rates()
+
+    second = await run_sync(CbrSource(), settings=settings(), cursor=first.cursor_after)
+
+    assert second.counts == first.counts
+    assert await stored_rates() == before
+    params = route_params(dynamic)[-1]
+    expected_start = date.fromisoformat(first.cursor_after or "") - timedelta(days=3)
+    assert params["date_req1"] == expected_start.strftime("%d/%m/%Y")
+
+
+async def test_no_foreign_currency_skips_the_network(app, mock_http, fixture_bytes, run_sync):
+    await make_account(currency="RUB")
+    daily, dynamic = mock_cbr(mock_http, fixture_bytes)
+
+    result = await run_sync(CbrSource(), settings=settings())
+
+    assert not daily.called and not dynamic.called
+    assert result.changed is False
+    assert result.counts == {"currencies": 0, "rates": 0}
+
+
+def test_long_ranges_are_chunked_by_year():
+    windows = split_range(date(2020, 1, 1), date(2023, 6, 1))
+    assert len(windows) == 4
+    assert windows[0][0] == date(2020, 1, 1)
+    assert windows[-1][1] == date(2023, 6, 1)
+    for start, end in windows:
+        assert (end - start).days < 366
+    # windows are contiguous, no day is fetched twice or skipped
+    for (_, end), (start, _) in pairwise(windows):
+        assert start == end + timedelta(days=1)
+    assert split_range(date(2026, 1, 2), date(2026, 1, 1)) == []