Source.sync(ctx) -> SyncResult пишет только raw_* и возвращает курсор; локи, журнал, ошибки и продвижение курсора берёт на себя worker/runner. ZenMoney читается единственным доступным способом — POST /v8/diff/ по serverTimestamp; токен живёт сутки, поэтому worker ротирует refresh_token через source_credential. Маппер всегда пересобирает core из полных raw_*, так что удаление в ZenMoney исчезает и у нас. ЦБ ходит мимо прокси (trust_env=False) и отдаёт cp1251 с делением на Nominal. Курсы только по рабочим дням — протяжку по календарю делает аналитика. Планировщик — APScheduler в отдельном процессе, на источник advisory-лок sync:<name>, чтобы ручной запуск не пересёкся с плановым.
164 lines
5.5 KiB
Python
164 lines
5.5 KiB
Python
"""CBR (Bank of Russia) official FX rates over the public XML scripts.
|
|
|
|
Two endpoints:
|
|
|
|
* `GET /scripts/XML_daily.asp?date_req=DD/MM/YYYY` — every quoted currency on one day;
|
|
used only for the `CharCode -> internal id` map ('USD' -> 'R01235').
|
|
* `GET /scripts/XML_dynamic.asp?date_req1=…&date_req2=…&VAL_NM_RQ=R01235` — one currency
|
|
over a date range, `<Record Date="01.09.2026"><Nominal>1</Nominal><Value>92,1234</Value>`.
|
|
|
|
Both answer windows-1251 XML with a decimal comma, and quote RUB per `Nominal` units
|
|
(100 for JPY, 10 for CNY at times). Values are parsed into `Decimal` and kept exactly as
|
|
printed, nominal included — dividing per unit happens later in `pricing/fx.py`.
|
|
|
|
NETWORK NOTE: `trust_env=False`. This host exports http_proxy/https_proxy/all_proxy, and
|
|
cbr.ru fails the TLS handshake through that proxy, so the CBR client must bypass the
|
|
environment entirely. (ZenMoney is the opposite case and keeps the proxy — see
|
|
`sources/zenmoney/client.py`.)
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import xml.etree.ElementTree as ET
|
|
from dataclasses import dataclass
|
|
from datetime import date, datetime, timedelta
|
|
from decimal import Decimal, InvalidOperation
|
|
|
|
import httpx
|
|
|
|
DAILY_URL = "https://www.cbr.ru/scripts/XML_daily.asp"
|
|
DYNAMIC_URL = "https://www.cbr.ru/scripts/XML_dynamic.asp"
|
|
ENCODING = "windows-1251"
|
|
TIMEOUT = 60.0
|
|
MAX_RANGE_DAYS = 366
|
|
"""CBR copes with longer ranges, but requests are chunked to at most a year to be polite."""
|
|
|
|
|
|
class CbrError(RuntimeError):
|
|
"""CBR refused or returned something unparsable."""
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class CbrQuote:
|
|
rate_date: date
|
|
nominal: int
|
|
value: Decimal
|
|
"""RUB per `nominal` units, exactly as CBR printed it."""
|
|
|
|
|
|
def new_http_client() -> httpx.AsyncClient:
|
|
# trust_env=False: see NETWORK NOTE above
|
|
return httpx.AsyncClient(trust_env=False, timeout=TIMEOUT)
|
|
|
|
|
|
def _fmt(day: date) -> str:
|
|
return day.strftime("%d/%m/%Y")
|
|
|
|
|
|
def _decimal(text: str | None) -> Decimal | None:
|
|
if not text:
|
|
return None
|
|
try:
|
|
return Decimal(text.replace("\xa0", "").replace(" ", "").replace(",", "."))
|
|
except InvalidOperation:
|
|
return None
|
|
|
|
|
|
def _parse_xml(payload: bytes) -> ET.Element:
|
|
try:
|
|
return ET.fromstring(payload.decode(ENCODING))
|
|
except (ET.ParseError, UnicodeDecodeError) as exc:
|
|
raise CbrError(f"CBR returned unparsable XML: {exc}") from exc
|
|
|
|
|
|
def parse_code_map(payload: bytes) -> dict[str, str]:
|
|
"""XML_daily -> {'USD': 'R01235', …}."""
|
|
root = _parse_xml(payload)
|
|
codes: dict[str, str] = {}
|
|
for node in root.iter("Valute"):
|
|
char_code = (node.findtext("CharCode") or "").strip().upper()
|
|
cbr_id = (node.get("ID") or "").strip()
|
|
if char_code and cbr_id:
|
|
codes[char_code] = cbr_id
|
|
return codes
|
|
|
|
|
|
def parse_dynamic(payload: bytes) -> list[CbrQuote]:
|
|
"""XML_dynamic -> quotes, nominal and value untouched."""
|
|
root = _parse_xml(payload)
|
|
quotes: list[CbrQuote] = []
|
|
for node in root.iter("Record"):
|
|
raw_date = node.get("Date")
|
|
value = _decimal(node.findtext("Value"))
|
|
if not raw_date or value is None:
|
|
continue
|
|
try:
|
|
rate_date = datetime.strptime(raw_date, "%d.%m.%Y").date()
|
|
except ValueError:
|
|
continue
|
|
nominal_text = (node.findtext("Nominal") or "1").strip()
|
|
try:
|
|
nominal = int(nominal_text.replace("\xa0", "").replace(" ", "")) or 1
|
|
except ValueError:
|
|
nominal = 1
|
|
quotes.append(CbrQuote(rate_date=rate_date, nominal=nominal, value=value))
|
|
return quotes
|
|
|
|
|
|
def split_range(start: date, end: date, max_days: int = MAX_RANGE_DAYS) -> list[tuple[date, date]]:
|
|
"""Chunk a long range into windows of at most `max_days` days."""
|
|
if start > end:
|
|
return []
|
|
windows: list[tuple[date, date]] = []
|
|
window_start = start
|
|
while window_start <= end:
|
|
window_end = min(end, window_start + timedelta(days=max_days - 1))
|
|
windows.append((window_start, window_end))
|
|
window_start = window_end + timedelta(days=1)
|
|
return windows
|
|
|
|
|
|
class CbrClient:
|
|
def __init__(self, http: httpx.AsyncClient | None = None) -> None:
|
|
self._http = http
|
|
self._owns_http = http is None
|
|
|
|
def _client(self) -> httpx.AsyncClient:
|
|
if self._http is None:
|
|
self._http = new_http_client()
|
|
return self._http
|
|
|
|
async def aclose(self) -> None:
|
|
if self._http is not None and self._owns_http:
|
|
await self._http.aclose()
|
|
self._http = None
|
|
|
|
async def __aenter__(self) -> CbrClient:
|
|
return self
|
|
|
|
async def __aexit__(self, *exc_info: object) -> None:
|
|
await self.aclose()
|
|
|
|
async def _get(self, url: str, params: dict[str, str]) -> bytes:
|
|
resp = await self._client().get(url, params=params)
|
|
if resp.status_code >= 400:
|
|
raise CbrError(f"{url} failed: HTTP {resp.status_code}")
|
|
return resp.content
|
|
|
|
async def code_map(self, on: date) -> dict[str, str]:
|
|
return parse_code_map(await self._get(DAILY_URL, {"date_req": _fmt(on)}))
|
|
|
|
async def quotes(self, cbr_id: str, start: date, end: date) -> list[CbrQuote]:
|
|
collected: list[CbrQuote] = []
|
|
for window_start, window_end in split_range(start, end):
|
|
payload = await self._get(
|
|
DYNAMIC_URL,
|
|
{
|
|
"date_req1": _fmt(window_start),
|
|
"date_req2": _fmt(window_end),
|
|
"VAL_NM_RQ": cbr_id,
|
|
},
|
|
)
|
|
collected.extend(parse_dynamic(payload))
|
|
return collected
|