diff --git a/backend/src/fintracker/sources/cbr/sync.py b/backend/src/fintracker/sources/cbr/sync.py index c1b7a9c..791a7bb 100644 --- a/backend/src/fintracker/sources/cbr/sync.py +++ b/backend/src/fintracker/sources/cbr/sync.py @@ -1,8 +1,14 @@ """The `cbr` source: official RUB rates for every currency the ledger actually uses. -What to fetch is derived from the data, not configured: the distinct currencies of -`account.currency`, `cash_txn.income_currency` and `cash_txn.outcome_currency` (RUB itself -is quoted as 1.0 by `pricing/fx.py`, so it is never requested). +What to fetch: `ALWAYS_QUOTED` (the dollar and the yuan) whatever the data says, plus every +currency the data uses — the distinct currencies of `account.currency`, `cash_txn.income_currency` +and `cash_txn.outcome_currency`. RUB itself is quoted as 1.0 by `pricing/fx.py`, so it is never +requested. The always-quoted pair exists because a currency that first appears in ZenMoney +between two runs would otherwise show «нет курса» until the next run picked it up. + +A currency with no stored rates yet is fetched from the start of the ledger, not from the shared +cursor: the cursor only knows what the OTHER currencies already have, and a new one needs the rates +for every past transaction it appears in. Date range: from the earliest transaction minus 7 days (or today − 30 days when there are no transactions yet) up to today — `analytics.today_local()`, the deployment timezone, the @@ -41,6 +47,8 @@ CURSOR_OVERLAP_DAYS = 3 BACKFILL_TOLERANCE_DAYS = 14 """How far the first stored quote may legitimately sit after the wanted start (holidays).""" NO_HISTORY_DAYS = 30 +ALWAYS_QUOTED = ("USD", "CNY") +"""Fetched every run, in use or not: the two foreign currencies the ledger meets first.""" CHUNK = 500 @@ -50,16 +58,9 @@ class CbrSource: async def sync(self, ctx: SyncContext) -> SyncResult: session = ctx.session today = today_local() - currencies = await currencies_in_use(session) - if not currencies: - log.info("cbr: only %s in use, nothing to fetch", BASE) - return SyncResult( - cursor_after=today.isoformat(), - counts={"currencies": 0, "rates": 0}, - changed=False, - ) - - start = await _start_date(session, ctx.cursor_before, today) + currencies = sorted(set(await currencies_in_use(session)) | set(ALWAYS_QUOTED)) + full_start = await _full_start(session, today) + start = await _start_date(session, ctx.cursor_before, today, full_start) warnings: list[str] = [] stored = 0 async with CbrClient() as client: @@ -70,7 +71,11 @@ class CbrSource: if not cbr_id: warnings.append(f"{ccy} is not quoted by CBR — skipped (crypto or metal?)") continue - quotes = await client.quotes(cbr_id, start, today) + # a currency the table has never seen needs the whole history, cursor or not + have = await session.scalar( + select(RawCbrRate.ccy).where(RawCbrRate.ccy == ccy).limit(1) + ) + quotes = await client.quotes(cbr_id, start if have else full_start, today) stored += await _store(session, ccy, quotes) fetched_currencies += 1 @@ -104,13 +109,19 @@ async def currencies_in_use(session: AsyncSession) -> list[str]: return sorted(code for code in found if code != BASE) -async def _start_date(session: AsyncSession, cursor: str | None, today: date) -> date: +async def _full_start(session: AsyncSession, today: date) -> date: + """Where the full history begins: a week before the first transaction, else 30 days back.""" first_txn, _ = await ledger_date_range(session) - full_start = ( + return ( first_txn - timedelta(days=LOOKBACK_DAYS) if first_txn is not None else today - timedelta(days=NO_HISTORY_DAYS) ) + + +async def _start_date( + session: AsyncSession, cursor: str | None, today: date, full_start: date +) -> date: cursor_date = _parse_date(cursor) if cursor_date is None: return full_start diff --git a/backend/tests/fixtures/cbr/daily.xml b/backend/tests/fixtures/cbr/daily.xml index 14620f2..76cbb9b 100644 --- a/backend/tests/fixtures/cbr/daily.xml +++ b/backend/tests/fixtures/cbr/daily.xml @@ -24,4 +24,12 @@ 101,4567 101,4567 + +156 +CNY +1 + +12,7654 +12,7654 + diff --git a/backend/tests/fixtures/cbr/dynamic_cny.xml b/backend/tests/fixtures/cbr/dynamic_cny.xml new file mode 100644 index 0000000..6618f00 --- /dev/null +++ b/backend/tests/fixtures/cbr/dynamic_cny.xml @@ -0,0 +1,18 @@ + + + +1 +12,7000 +12,7 + + +1 +12,7300 +12,73 + + +1 +12,7654 +12,7654 + + diff --git a/backend/tests/sources/test_cbr.py b/backend/tests/sources/test_cbr.py index c6acb2c..6af3e8c 100644 --- a/backend/tests/sources/test_cbr.py +++ b/backend/tests/sources/test_cbr.py @@ -19,6 +19,7 @@ from fintracker.sources.cbr.sync import CbrSource USD_ID = "R01235" JPY_ID = "R01820" +CNY_ID = "R01375" def settings() -> Settings: @@ -39,7 +40,11 @@ def mock_cbr(mock_http, fixture_bytes, *, dynamic: dict[str, str] | None = None) daily = mock_http.get(DAILY_URL).mock( return_value=httpx.Response(200, content=fixture_bytes("cbr", "daily.xml")) ) - files = dynamic or {USD_ID: "dynamic_usd.xml", JPY_ID: "dynamic_jpy.xml"} + files = dynamic or { + USD_ID: "dynamic_usd.xml", + JPY_ID: "dynamic_jpy.xml", + CNY_ID: "dynamic_cny.xml", + } def _dynamic(request: httpx.Request) -> httpx.Response: cbr_id = request.url.params.get("VAL_NM_RQ", "") @@ -51,17 +56,19 @@ def mock_cbr(mock_http, fixture_bytes, *, dynamic: dict[str, str] | None = None) return daily, mock_http.get(DYNAMIC_URL).mock(side_effect=_dynamic) -async def test_only_currencies_in_use_are_requested(app, mock_http, fixture_bytes, run_sync): +async def test_currencies_in_use_are_requested_besides_the_always_quoted_pair( + app, mock_http, fixture_bytes, run_sync +): await make_account(currency="RUB") - await make_account(currency="USD", source_id="usd-cash") - await make_txn(date(2026, 9, 1), outcome="100", outcome_currency="USD") + await make_account(currency="JPY", source_id="jpy-cash") + await make_txn(date(2026, 9, 1), outcome="100", outcome_currency="JPY") _, dynamic = mock_cbr(mock_http, fixture_bytes) result = await run_sync(CbrSource(), settings=settings()) requested = {params["VAL_NM_RQ"] for params in route_params(dynamic)} - assert requested == {USD_ID} # EUR and JPY are quoted by CBR but unused here - assert result.counts == {"currencies": 1, "rates": 3} + assert requested == {USD_ID, CNY_ID, JPY_ID} # EUR is quoted by CBR, but nobody uses it + assert result.counts["currencies"] == 3 assert result.warnings == [] assert result.changed is True assert result.cursor_after is not None @@ -88,13 +95,13 @@ async def test_currency_cbr_does_not_quote_is_a_warning(app, mock_http, fixture_ _, dynamic = mock_cbr(mock_http, fixture_bytes) result = await run_sync(CbrSource(), settings=settings()) - assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID} + assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID, CNY_ID} assert sorted(result.warnings) == [ "BTC is not quoted by CBR — skipped (crypto or metal?)", "XAU is not quoted by CBR — skipped (crypto or metal?)", ] - assert result.counts["currencies"] == 1 - assert result.counts["rates"] == 3 + assert result.counts["currencies"] == 2 + assert result.counts["rates"] == 6 async def test_range_starts_before_the_first_transaction(app, mock_http, fixture_bytes, run_sync): @@ -129,15 +136,43 @@ async def test_cursor_shortens_the_range_and_reruns_are_idempotent( assert params["date_req1"] == expected_start.strftime("%d/%m/%Y") -async def test_no_foreign_currency_skips_the_network(app, mock_http, fixture_bytes, run_sync): +async def test_the_dollar_and_the_yuan_are_fetched_even_with_only_roubles( + app, mock_http, fixture_bytes, run_sync +): + """A currency that shows up in ZenMoney between two runs must already have its rate.""" await make_account(currency="RUB") daily, dynamic = mock_cbr(mock_http, fixture_bytes) result = await run_sync(CbrSource(), settings=settings()) - assert not daily.called and not dynamic.called - assert result.changed is False - assert result.counts == {"currencies": 0, "rates": 0} + assert daily.called + assert {params["VAL_NM_RQ"] for params in route_params(dynamic)} == {USD_ID, CNY_ID} + assert result.changed is True + assert result.counts == {"currencies": 2, "rates": 6} + rates = await stored_rates() + assert rates[("CNY", "2026-09-03")] == (1, Decimal("12.7654")) + + +async def test_a_currency_new_to_the_table_is_backfilled_from_the_start_of_the_ledger( + app, mock_http, fixture_bytes, run_sync +): + """The cursor only knows what the other currencies have; a new one needs its whole past.""" + first = today_local() - timedelta(days=10) + await make_account(currency="USD", source_id="usd") + await make_txn(first, outcome="10", outcome_currency="USD") + _, dynamic = mock_cbr(mock_http, fixture_bytes) + earlier = await run_sync(CbrSource(), settings=settings()) + + await make_account(currency="JPY", source_id="jpy") + await make_txn(first + timedelta(days=2), outcome="1000", outcome_currency="JPY") + seen = len(dynamic.calls) + await run_sync(CbrSource(), settings=settings(), cursor=earlier.cursor_after) + + starts = {p["VAL_NM_RQ"]: p["date_req1"] for p in route_params(dynamic)[seen:]} + incremental = date.fromisoformat(earlier.cursor_after or "") - timedelta(days=3) + assert starts[JPY_ID] == (first - timedelta(days=7)).strftime("%d/%m/%Y") + assert starts[USD_ID] == incremental.strftime("%d/%m/%Y") + assert starts[CNY_ID] == incremental.strftime("%d/%m/%Y") def test_long_ranges_are_chunked_by_year():