feat(cbr): USD и CNY тянутся всегда, новая валюта — с начала леджера
Валюта, впервые появившаяся в ZenMoney между двумя запусками, раньше показывала «нет курса» до следующего синка. Для валюты без сохранённых курсов история берётся с начала леджера, а не от общего курсора.
This commit is contained in:
@@ -1,8 +1,14 @@
|
|||||||
"""The `cbr` source: official RUB rates for every currency the ledger actually uses.
|
"""The `cbr` source: official RUB rates for every currency the ledger actually uses.
|
||||||
|
|
||||||
What to fetch is derived from the data, not configured: the distinct currencies of
|
What to fetch: `ALWAYS_QUOTED` (the dollar and the yuan) whatever the data says, plus every
|
||||||
`account.currency`, `cash_txn.income_currency` and `cash_txn.outcome_currency` (RUB itself
|
currency the data uses — the distinct currencies of `account.currency`, `cash_txn.income_currency`
|
||||||
is quoted as 1.0 by `pricing/fx.py`, so it is never requested).
|
and `cash_txn.outcome_currency`. RUB itself is quoted as 1.0 by `pricing/fx.py`, so it is never
|
||||||
|
requested. The always-quoted pair exists because a currency that first appears in ZenMoney
|
||||||
|
between two runs would otherwise show «нет курса» until the next run picked it up.
|
||||||
|
|
||||||
|
A currency with no stored rates yet is fetched from the start of the ledger, not from the shared
|
||||||
|
cursor: the cursor only knows what the OTHER currencies already have, and a new one needs the rates
|
||||||
|
for every past transaction it appears in.
|
||||||
|
|
||||||
Date range: from the earliest transaction minus 7 days (or today − 30 days when there are
|
Date range: from the earliest transaction minus 7 days (or today − 30 days when there are
|
||||||
no transactions yet) up to today — `analytics.today_local()`, the deployment timezone, the
|
no transactions yet) up to today — `analytics.today_local()`, the deployment timezone, the
|
||||||
@@ -41,6 +47,8 @@ CURSOR_OVERLAP_DAYS = 3
|
|||||||
BACKFILL_TOLERANCE_DAYS = 14
|
BACKFILL_TOLERANCE_DAYS = 14
|
||||||
"""How far the first stored quote may legitimately sit after the wanted start (holidays)."""
|
"""How far the first stored quote may legitimately sit after the wanted start (holidays)."""
|
||||||
NO_HISTORY_DAYS = 30
|
NO_HISTORY_DAYS = 30
|
||||||
|
ALWAYS_QUOTED = ("USD", "CNY")
|
||||||
|
"""Fetched every run, in use or not: the two foreign currencies the ledger meets first."""
|
||||||
CHUNK = 500
|
CHUNK = 500
|
||||||
|
|
||||||
|
|
||||||
@@ -50,16 +58,9 @@ class CbrSource:
|
|||||||
async def sync(self, ctx: SyncContext) -> SyncResult:
|
async def sync(self, ctx: SyncContext) -> SyncResult:
|
||||||
session = ctx.session
|
session = ctx.session
|
||||||
today = today_local()
|
today = today_local()
|
||||||
currencies = await currencies_in_use(session)
|
currencies = sorted(set(await currencies_in_use(session)) | set(ALWAYS_QUOTED))
|
||||||
if not currencies:
|
full_start = await _full_start(session, today)
|
||||||
log.info("cbr: only %s in use, nothing to fetch", BASE)
|
start = await _start_date(session, ctx.cursor_before, today, full_start)
|
||||||
return SyncResult(
|
|
||||||
cursor_after=today.isoformat(),
|
|
||||||
counts={"currencies": 0, "rates": 0},
|
|
||||||
changed=False,
|
|
||||||
)
|
|
||||||
|
|
||||||
start = await _start_date(session, ctx.cursor_before, today)
|
|
||||||
warnings: list[str] = []
|
warnings: list[str] = []
|
||||||
stored = 0
|
stored = 0
|
||||||
async with CbrClient() as client:
|
async with CbrClient() as client:
|
||||||
@@ -70,7 +71,11 @@ class CbrSource:
|
|||||||
if not cbr_id:
|
if not cbr_id:
|
||||||
warnings.append(f"{ccy} is not quoted by CBR — skipped (crypto or metal?)")
|
warnings.append(f"{ccy} is not quoted by CBR — skipped (crypto or metal?)")
|
||||||
continue
|
continue
|
||||||
quotes = await client.quotes(cbr_id, start, today)
|
# a currency the table has never seen needs the whole history, cursor or not
|
||||||
|
have = await session.scalar(
|
||||||
|
select(RawCbrRate.ccy).where(RawCbrRate.ccy == ccy).limit(1)
|
||||||
|
)
|
||||||
|
quotes = await client.quotes(cbr_id, start if have else full_start, today)
|
||||||
stored += await _store(session, ccy, quotes)
|
stored += await _store(session, ccy, quotes)
|
||||||
fetched_currencies += 1
|
fetched_currencies += 1
|
||||||
|
|
||||||
@@ -104,13 +109,19 @@ async def currencies_in_use(session: AsyncSession) -> list[str]:
|
|||||||
return sorted(code for code in found if code != BASE)
|
return sorted(code for code in found if code != BASE)
|
||||||
|
|
||||||
|
|
||||||
async def _start_date(session: AsyncSession, cursor: str | None, today: date) -> date:
|
async def _full_start(session: AsyncSession, today: date) -> date:
|
||||||
|
"""Where the full history begins: a week before the first transaction, else 30 days back."""
|
||||||
first_txn, _ = await ledger_date_range(session)
|
first_txn, _ = await ledger_date_range(session)
|
||||||
full_start = (
|
return (
|
||||||
first_txn - timedelta(days=LOOKBACK_DAYS)
|
first_txn - timedelta(days=LOOKBACK_DAYS)
|
||||||
if first_txn is not None
|
if first_txn is not None
|
||||||
else today - timedelta(days=NO_HISTORY_DAYS)
|
else today - timedelta(days=NO_HISTORY_DAYS)
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
async def _start_date(
|
||||||
|
session: AsyncSession, cursor: str | None, today: date, full_start: date
|
||||||
|
) -> date:
|
||||||
cursor_date = _parse_date(cursor)
|
cursor_date = _parse_date(cursor)
|
||||||
if cursor_date is None:
|
if cursor_date is None:
|
||||||
return full_start
|
return full_start
|
||||||
|
|||||||
Vendored
+8
@@ -24,4 +24,12 @@
|
|||||||
<Value>101,4567</Value>
|
<Value>101,4567</Value>
|
||||||
<VunitRate>101,4567</VunitRate>
|
<VunitRate>101,4567</VunitRate>
|
||||||
</Valute>
|
</Valute>
|
||||||
|
<Valute ID="R01375">
|
||||||
|
<NumCode>156</NumCode>
|
||||||
|
<CharCode>CNY</CharCode>
|
||||||
|
<Nominal>1</Nominal>
|
||||||
|
<Name>Êèòàéñêèé þàíü</Name>
|
||||||
|
<Value>12,7654</Value>
|
||||||
|
<VunitRate>12,7654</VunitRate>
|
||||||
|
</Valute>
|
||||||
</ValCurs>
|
</ValCurs>
|
||||||
|
|||||||
+18
@@ -0,0 +1,18 @@
|
|||||||
|
<?xml version="1.0" encoding="windows-1251"?>
|
||||||
|
<ValCurs ID="R01375" DateRange1="01.09.2026" DateRange2="03.09.2026" name="Foreign Currency Market Dynamic">
|
||||||
|
<Record Date="01.09.2026" Id="R01375">
|
||||||
|
<Nominal>1</Nominal>
|
||||||
|
<Value>12,7000</Value>
|
||||||
|
<VunitRate>12,7</VunitRate>
|
||||||
|
</Record>
|
||||||
|
<Record Date="02.09.2026" Id="R01375">
|
||||||
|
<Nominal>1</Nominal>
|
||||||
|
<Value>12,7300</Value>
|
||||||
|
<VunitRate>12,73</VunitRate>
|
||||||
|
</Record>
|
||||||
|
<Record Date="03.09.2026" Id="R01375">
|
||||||
|
<Nominal>1</Nominal>
|
||||||
|
<Value>12,7654</Value>
|
||||||
|
<VunitRate>12,7654</VunitRate>
|
||||||
|
</Record>
|
||||||
|
</ValCurs>
|
||||||
@@ -19,6 +19,7 @@ from fintracker.sources.cbr.sync import CbrSource
|
|||||||
|
|
||||||
USD_ID = "R01235"
|
USD_ID = "R01235"
|
||||||
JPY_ID = "R01820"
|
JPY_ID = "R01820"
|
||||||
|
CNY_ID = "R01375"
|
||||||
|
|
||||||
|
|
||||||
def settings() -> Settings:
|
def settings() -> Settings:
|
||||||
@@ -39,7 +40,11 @@ def mock_cbr(mock_http, fixture_bytes, *, dynamic: dict[str, str] | None = None)
|
|||||||
daily = mock_http.get(DAILY_URL).mock(
|
daily = mock_http.get(DAILY_URL).mock(
|
||||||
return_value=httpx.Response(200, content=fixture_bytes("cbr", "daily.xml"))
|
return_value=httpx.Response(200, content=fixture_bytes("cbr", "daily.xml"))
|
||||||
)
|
)
|
||||||
files = dynamic or {USD_ID: "dynamic_usd.xml", JPY_ID: "dynamic_jpy.xml"}
|
files = dynamic or {
|
||||||
|
USD_ID: "dynamic_usd.xml",
|
||||||
|
JPY_ID: "dynamic_jpy.xml",
|
||||||
|
CNY_ID: "dynamic_cny.xml",
|
||||||
|
}
|
||||||
|
|
||||||
def _dynamic(request: httpx.Request) -> httpx.Response:
|
def _dynamic(request: httpx.Request) -> httpx.Response:
|
||||||
cbr_id = request.url.params.get("VAL_NM_RQ", "")
|
cbr_id = request.url.params.get("VAL_NM_RQ", "")
|
||||||
@@ -51,17 +56,19 @@ def mock_cbr(mock_http, fixture_bytes, *, dynamic: dict[str, str] | None = None)
|
|||||||
return daily, mock_http.get(DYNAMIC_URL).mock(side_effect=_dynamic)
|
return daily, mock_http.get(DYNAMIC_URL).mock(side_effect=_dynamic)
|
||||||
|
|
||||||
|
|
||||||
async def test_only_currencies_in_use_are_requested(app, mock_http, fixture_bytes, run_sync):
|
async def test_currencies_in_use_are_requested_besides_the_always_quoted_pair(
|
||||||
|
app, mock_http, fixture_bytes, run_sync
|
||||||
|
):
|
||||||
await make_account(currency="RUB")
|
await make_account(currency="RUB")
|
||||||
await make_account(currency="USD", source_id="usd-cash")
|
await make_account(currency="JPY", source_id="jpy-cash")
|
||||||
await make_txn(date(2026, 9, 1), outcome="100", outcome_currency="USD")
|
await make_txn(date(2026, 9, 1), outcome="100", outcome_currency="JPY")
|
||||||
|
|
||||||
_, dynamic = mock_cbr(mock_http, fixture_bytes)
|
_, dynamic = mock_cbr(mock_http, fixture_bytes)
|
||||||
result = await run_sync(CbrSource(), settings=settings())
|
result = await run_sync(CbrSource(), settings=settings())
|
||||||
|
|
||||||
requested = {params["VAL_NM_RQ"] for params in route_params(dynamic)}
|
requested = {params["VAL_NM_RQ"] for params in route_params(dynamic)}
|
||||||
assert requested == {USD_ID} # EUR and JPY are quoted by CBR but unused here
|
assert requested == {USD_ID, CNY_ID, JPY_ID} # EUR is quoted by CBR, but nobody uses it
|
||||||
assert result.counts == {"currencies": 1, "rates": 3}
|
assert result.counts["currencies"] == 3
|
||||||
assert result.warnings == []
|
assert result.warnings == []
|
||||||
assert result.changed is True
|
assert result.changed is True
|
||||||
assert result.cursor_after is not None
|
assert result.cursor_after is not None
|
||||||
@@ -88,13 +95,13 @@ async def test_currency_cbr_does_not_quote_is_a_warning(app, mock_http, fixture_
|
|||||||
_, dynamic = mock_cbr(mock_http, fixture_bytes)
|
_, dynamic = mock_cbr(mock_http, fixture_bytes)
|
||||||
result = await run_sync(CbrSource(), settings=settings())
|
result = await run_sync(CbrSource(), settings=settings())
|
||||||
|
|
||||||
assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID}
|
assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID, CNY_ID}
|
||||||
assert sorted(result.warnings) == [
|
assert sorted(result.warnings) == [
|
||||||
"BTC is not quoted by CBR — skipped (crypto or metal?)",
|
"BTC is not quoted by CBR — skipped (crypto or metal?)",
|
||||||
"XAU is not quoted by CBR — skipped (crypto or metal?)",
|
"XAU is not quoted by CBR — skipped (crypto or metal?)",
|
||||||
]
|
]
|
||||||
assert result.counts["currencies"] == 1
|
assert result.counts["currencies"] == 2
|
||||||
assert result.counts["rates"] == 3
|
assert result.counts["rates"] == 6
|
||||||
|
|
||||||
|
|
||||||
async def test_range_starts_before_the_first_transaction(app, mock_http, fixture_bytes, run_sync):
|
async def test_range_starts_before_the_first_transaction(app, mock_http, fixture_bytes, run_sync):
|
||||||
@@ -129,15 +136,43 @@ async def test_cursor_shortens_the_range_and_reruns_are_idempotent(
|
|||||||
assert params["date_req1"] == expected_start.strftime("%d/%m/%Y")
|
assert params["date_req1"] == expected_start.strftime("%d/%m/%Y")
|
||||||
|
|
||||||
|
|
||||||
async def test_no_foreign_currency_skips_the_network(app, mock_http, fixture_bytes, run_sync):
|
async def test_the_dollar_and_the_yuan_are_fetched_even_with_only_roubles(
|
||||||
|
app, mock_http, fixture_bytes, run_sync
|
||||||
|
):
|
||||||
|
"""A currency that shows up in ZenMoney between two runs must already have its rate."""
|
||||||
await make_account(currency="RUB")
|
await make_account(currency="RUB")
|
||||||
daily, dynamic = mock_cbr(mock_http, fixture_bytes)
|
daily, dynamic = mock_cbr(mock_http, fixture_bytes)
|
||||||
|
|
||||||
result = await run_sync(CbrSource(), settings=settings())
|
result = await run_sync(CbrSource(), settings=settings())
|
||||||
|
|
||||||
assert not daily.called and not dynamic.called
|
assert daily.called
|
||||||
assert result.changed is False
|
assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID, CNY_ID}
|
||||||
assert result.counts == {"currencies": 0, "rates": 0}
|
assert result.changed is True
|
||||||
|
assert result.counts == {"currencies": 2, "rates": 6}
|
||||||
|
rates = await stored_rates()
|
||||||
|
assert rates[("CNY", "2026-09-03")] == (1, Decimal("12.7654"))
|
||||||
|
|
||||||
|
|
||||||
|
async def test_a_currency_new_to_the_table_is_backfilled_from_the_start_of_the_ledger(
|
||||||
|
app, mock_http, fixture_bytes, run_sync
|
||||||
|
):
|
||||||
|
"""The cursor only knows what the other currencies have; a new one needs its whole past."""
|
||||||
|
first = today_local() - timedelta(days=10)
|
||||||
|
await make_account(currency="USD", source_id="usd")
|
||||||
|
await make_txn(first, outcome="10", outcome_currency="USD")
|
||||||
|
_, dynamic = mock_cbr(mock_http, fixture_bytes)
|
||||||
|
earlier = await run_sync(CbrSource(), settings=settings())
|
||||||
|
|
||||||
|
await make_account(currency="JPY", source_id="jpy")
|
||||||
|
await make_txn(first + timedelta(days=2), outcome="1000", outcome_currency="JPY")
|
||||||
|
seen = len(dynamic.calls)
|
||||||
|
await run_sync(CbrSource(), settings=settings(), cursor=earlier.cursor_after)
|
||||||
|
|
||||||
|
starts = {p["VAL_NM_RQ"]: p["date_req1"] for p in route_params(dynamic)[seen:]}
|
||||||
|
incremental = date.fromisoformat(earlier.cursor_after or "") - timedelta(days=3)
|
||||||
|
assert starts[JPY_ID] == (first - timedelta(days=7)).strftime("%d/%m/%Y")
|
||||||
|
assert starts[USD_ID] == incremental.strftime("%d/%m/%Y")
|
||||||
|
assert starts[CNY_ID] == incremental.strftime("%d/%m/%Y")
|
||||||
|
|
||||||
|
|
||||||
def test_long_ranges_are_chunked_by_year():
|
def test_long_ranges_are_chunked_by_year():
|
||||||
|
|||||||
Reference in New Issue
Block a user