feat(sources): контракт источников, worker и синк ZenMoney + ЦБ

Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
2026-09-18 13:43:49 +03:00
parent 3fc7a954b9
commit c55fe19e48
31 changed files with 3713 additions and 0 deletions
@@ -0,0 +1,163 @@
+"""CBR (Bank of Russia) official FX rates over the public XML scripts.
+
+Two endpoints:
+
+* `GET /scripts/XML_daily.asp?date_req=DD/MM/YYYY` — every quoted currency on one day;
+  used only for the `CharCode -> internal id` map ('USD' -> 'R01235').
+* `GET /scripts/XML_dynamic.asp?date_req1=…&date_req2=…&VAL_NM_RQ=R01235` — one currency
+  over a date range, `<Record Date="01.09.2026"><Nominal>1</Nominal><Value>92,1234</Value>`.
+
+Both answer windows-1251 XML with a decimal comma, and quote RUB per `Nominal` units
+(100 for JPY, 10 for CNY at times). Values are parsed into `Decimal` and kept exactly as
+printed, nominal included — dividing per unit happens later in `pricing/fx.py`.
+
+NETWORK NOTE: `trust_env=False`. This host exports http_proxy/https_proxy/all_proxy, and
+cbr.ru fails the TLS handshake through that proxy, so the CBR client must bypass the
+environment entirely. (ZenMoney is the opposite case and keeps the proxy — see
+`sources/zenmoney/client.py`.)
+"""
+
+from __future__ import annotations
+
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from datetime import date, datetime, timedelta
+from decimal import Decimal, InvalidOperation
+
+import httpx
+
+DAILY_URL = "https://www.cbr.ru/scripts/XML_daily.asp"
+DYNAMIC_URL = "https://www.cbr.ru/scripts/XML_dynamic.asp"
+ENCODING = "windows-1251"
+TIMEOUT = 60.0
+MAX_RANGE_DAYS = 366
+"""CBR copes with longer ranges, but requests are chunked to at most a year to be polite."""
+
+
+class CbrError(RuntimeError):
+    """CBR refused or returned something unparsable."""
+
+
+@dataclass(frozen=True)
+class CbrQuote:
+    rate_date: date
+    nominal: int
+    value: Decimal
+    """RUB per `nominal` units, exactly as CBR printed it."""
+
+
+def new_http_client() -> httpx.AsyncClient:
+    # trust_env=False: see NETWORK NOTE above
+    return httpx.AsyncClient(trust_env=False, timeout=TIMEOUT)
+
+
+def _fmt(day: date) -> str:
+    return day.strftime("%d/%m/%Y")
+
+
+def _decimal(text: str | None) -> Decimal | None:
+    if not text:
+        return None
+    try:
+        return Decimal(text.replace("\xa0", "").replace(" ", "").replace(",", "."))
+    except InvalidOperation:
+        return None
+
+
+def _parse_xml(payload: bytes) -> ET.Element:
+    try:
+        return ET.fromstring(payload.decode(ENCODING))
+    except (ET.ParseError, UnicodeDecodeError) as exc:
+        raise CbrError(f"CBR returned unparsable XML: {exc}") from exc
+
+
+def parse_code_map(payload: bytes) -> dict[str, str]:
+    """XML_daily -> {'USD': 'R01235', …}."""
+    root = _parse_xml(payload)
+    codes: dict[str, str] = {}
+    for node in root.iter("Valute"):
+        char_code = (node.findtext("CharCode") or "").strip().upper()
+        cbr_id = (node.get("ID") or "").strip()
+        if char_code and cbr_id:
+            codes[char_code] = cbr_id
+    return codes
+
+
+def parse_dynamic(payload: bytes) -> list[CbrQuote]:
+    """XML_dynamic -> quotes, nominal and value untouched."""
+    root = _parse_xml(payload)
+    quotes: list[CbrQuote] = []
+    for node in root.iter("Record"):
+        raw_date = node.get("Date")
+        value = _decimal(node.findtext("Value"))
+        if not raw_date or value is None:
+            continue
+        try:
+            rate_date = datetime.strptime(raw_date, "%d.%m.%Y").date()
+        except ValueError:
+            continue
+        nominal_text = (node.findtext("Nominal") or "1").strip()
+        try:
+            nominal = int(nominal_text.replace("\xa0", "").replace(" ", "")) or 1
+        except ValueError:
+            nominal = 1
+        quotes.append(CbrQuote(rate_date=rate_date, nominal=nominal, value=value))
+    return quotes
+
+
+def split_range(start: date, end: date, max_days: int = MAX_RANGE_DAYS) -> list[tuple[date, date]]:
+    """Chunk a long range into windows of at most `max_days` days."""
+    if start > end:
+        return []
+    windows: list[tuple[date, date]] = []
+    window_start = start
+    while window_start <= end:
+        window_end = min(end, window_start + timedelta(days=max_days - 1))
+        windows.append((window_start, window_end))
+        window_start = window_end + timedelta(days=1)
+    return windows
+
+
+class CbrClient:
+    def __init__(self, http: httpx.AsyncClient | None = None) -> None:
+        self._http = http
+        self._owns_http = http is None
+
+    def _client(self) -> httpx.AsyncClient:
+        if self._http is None:
+            self._http = new_http_client()
+        return self._http
+
+    async def aclose(self) -> None:
+        if self._http is not None and self._owns_http:
+            await self._http.aclose()
+            self._http = None
+
+    async def __aenter__(self) -> CbrClient:
+        return self
+
+    async def __aexit__(self, *exc_info: object) -> None:
+        await self.aclose()
+
+    async def _get(self, url: str, params: dict[str, str]) -> bytes:
+        resp = await self._client().get(url, params=params)
+        if resp.status_code >= 400:
+            raise CbrError(f"{url} failed: HTTP {resp.status_code}")
+        return resp.content
+
+    async def code_map(self, on: date) -> dict[str, str]:
+        return parse_code_map(await self._get(DAILY_URL, {"date_req": _fmt(on)}))
+
+    async def quotes(self, cbr_id: str, start: date, end: date) -> list[CbrQuote]:
+        collected: list[CbrQuote] = []
+        for window_start, window_end in split_range(start, end):
+            payload = await self._get(
+                DYNAMIC_URL,
+                {
+                    "date_req1": _fmt(window_start),
+                    "date_req2": _fmt(window_end),
+                    "VAL_NM_RQ": cbr_id,
+                },
+            )
+            collected.extend(parse_dynamic(payload))
+        return collected