"""MOEX sync: which window each instrument is asked for. The bug these cover: with one cursor for the whole source, a paper that entered the ledger after the first run was only ever fetched from the cursor, so the stretch between the day it was first held and that run stayed empty forever — and the valuation series had no price for days the portfolio demonstrably held it. """ from __future__ import annotations from datetime import date, timedelta import httpx import pytest from sqlalchemy import func, select from factories import make_account, make_event, make_instrument, make_price from fintracker.analytics import today_local from fintracker.config import Settings from fintracker.db import get_sessionmaker from fintracker.models import EventKind, PriceCoverage, PriceDaily from fintracker.sources.moex.client import BoardInfo from fintracker.sources.moex.sync import ( OVERLAP_DAYS, Coverage, MoexSource, Target, choose_board, fetch_window, history_legs, needs_backfill, ) ISS = "https://iss.moex.com/iss" TODAY = today_local() def monday_back(days: int) -> date: """A weekday roughly `days` ago — the mocked ISS answers on trading days only.""" d = TODAY - timedelta(days=days) return d - timedelta(days=d.isoweekday() - 1) def settings() -> Settings: return Settings() def target(since: date) -> Target: from fintracker.models import AssetClass return Target( instrument_id=1, secid="SBER", since=since, nominal=None, board="TQBR", exchange="shares", asset_class=AssetClass.share, ) BOARD_COLUMNS = [ "secid", "boardid", "market", "engine", "is_primary", "history_from", "history_till", ] def mock_iss(mock_http, boards: list[list] | None = None): """ISS with a history that answers exactly the window it was asked for.""" mock_http.get(url__startswith=f"{ISS}/securities/").mock( return_value=httpx.Response( 200, json={ "boards": { "columns": BOARD_COLUMNS, "data": boards or [["SBER", "TQBR", "shares", "stock", 1, "2013-03-25", TODAY.isoformat()]], } }, ) ) mock_http.get(url__startswith=f"{ISS}/engines/").mock( return_value=httpx.Response( 200, json={"marketdata": {"columns": ["LAST"], "data": [[275.65]]}} ) ) def _history(request: httpx.Request) -> httpx.Response: since = date.fromisoformat(request.url.params["from"]) until = date.fromisoformat(request.url.params["till"]) start = int(request.url.params.get("start", 0)) days = [ since + timedelta(days=i) for i in range((until - since).days + 1) if (since + timedelta(days=i)).isoweekday() < 6 ][start : start + 100] return httpx.Response( 200, json={ "history": { "columns": ["TRADEDATE", "CLOSE", "CURRENCYID"], "data": [[d.isoformat(), 100.0, "SUR"] for d in days], } }, ) return mock_http.get(url__startswith=f"{ISS}/history").mock(side_effect=_history) async def seed(*, first_held: date, priced: list[date], asked_from: date | None) -> int: """A ledger position, the prices already stored for it, and how deep we have asked.""" account_id = await make_account(name="Брокерский") instrument_id = await make_instrument(ticker="SBER") await make_event( first_held, account_id=account_id, kind=EventKind.buy, instrument_id=instrument_id, quantity="10", price="100", amount="-1000", ) for d in priced: await make_price(d, instrument_id=instrument_id, close="100") if asked_from is not None: async with get_sessionmaker()() as session: session.add( PriceCoverage(instrument_id=instrument_id, source="moex", history_from=asked_from) ) await session.commit() return instrument_id def legs(route) -> list[tuple[str, date, date]]: """One entry per (board, window) asked for, pagination collapsed.""" seen = [] for call in route.calls: params = call.request.url.params board = call.request.url.path.split("/boards/")[1].split("/")[0] leg = (board, date.fromisoformat(params["from"]), date.fromisoformat(params["till"])) if leg not in seen: seen.append(leg) return seen def windows(route) -> list[tuple[date, date]]: """One entry per history request, pagination collapsed.""" seen = [] for call in route.calls: params = call.request.url.params window = (date.fromisoformat(params["from"]), date.fromisoformat(params["till"])) if window not in seen: seen.append(window) return seen async def stored_days(instrument_id: int) -> tuple[int, date, date]: async with get_sessionmaker()() as session: row = ( await session.execute( select(func.count(), func.min(PriceDaily.d), func.max(PriceDaily.d)).where( PriceDaily.instrument_id == instrument_id ) ) ).one() return row[0], row[1], row[2] @pytest.mark.parametrize( ("history_from", "expected_backfill"), [(None, True), (date(2026, 1, 1), True), (date(2025, 1, 1), False)], ) def test_backfill_is_due_until_the_first_held_day_has_been_asked_for( history_from, expected_backfill ): coverage = Coverage(last=TODAY, history_from=history_from) assert needs_backfill(target(date(2025, 1, 1)), coverage) is expected_backfill def test_a_settled_history_only_re_reads_the_tail(): """ISS revises the settlement price after the close, so the last days are read again.""" since = TODAY - timedelta(days=400) coverage = Coverage(last=TODAY - timedelta(days=1), history_from=since) assert fetch_window(target(since), coverage, TODAY) == ( TODAY - timedelta(days=1 + OVERLAP_DAYS), TODAY, ) def test_a_paper_without_any_price_is_asked_from_the_day_it_was_held(): since = TODAY - timedelta(days=30) assert fetch_window(target(since), Coverage(), TODAY) == (since, TODAY) async def test_a_hole_in_the_past_is_pulled_back_to_the_first_held_day(app, mock_http, run_sync): """The TBRU@ case: held since spring, priced only from the day the source first saw it.""" first_held = monday_back(60) recent = [TODAY - timedelta(days=n) for n in (3, 2, 1)] instrument_id = await seed(first_held=first_held, priced=recent, asked_from=recent[0]) history = mock_iss(mock_http) result = await run_sync(MoexSource(), settings=settings()) assert windows(history) == [(first_held, TODAY)] count, low, high = await stored_days(instrument_id) assert low == first_held assert high >= recent[-1] assert count > len(recent) assert result.counts["backfilled"] == 1 assert result.warnings == [] async def test_a_complete_history_only_re_reads_the_overlap_window(app, mock_http, run_sync): first_held = monday_back(60) priced = [ first_held + timedelta(days=n) for n in range(61) if (first_held + timedelta(days=n)).isoweekday() < 6 ] await seed(first_held=first_held, priced=priced, asked_from=first_held) history = mock_iss(mock_http) result = await run_sync(MoexSource(), settings=settings()) assert windows(history) == [(priced[-1] - timedelta(days=OVERLAP_DAYS), TODAY)] assert result.counts["backfilled"] == 0 async def test_a_second_run_backfills_once_and_stores_no_duplicates(app, mock_http, run_sync): first_held = monday_back(60) instrument_id = await seed( first_held=first_held, priced=[TODAY - timedelta(days=1)], asked_from=None ) history = mock_iss(mock_http) first = await run_sync(MoexSource(), settings=settings()) after_first, _, _ = await stored_days(instrument_id) second = await run_sync(MoexSource(), settings=settings()) after_second, _, _ = await stored_days(instrument_id) assert first.counts["backfilled"] == 1 assert second.counts["backfilled"] == 0 # asking once is what the coverage row remembers assert windows(history)[-1][0] > first_held assert after_second == after_first # (instrument_id, d) is the key: the sync upserts async def test_a_long_break_inside_a_settled_history_is_reported(app, mock_http, run_sync): """A paper that stopped trading and one that failed to download look alike in the data — so the gap is surfaced as a warning instead of being silently re-fetched every run.""" first_held = monday_back(120) priced = [first_held, first_held + timedelta(days=1), TODAY - timedelta(days=1)] await seed(first_held=first_held, priced=priced, asked_from=first_held) mock_iss(mock_http) result = await run_sync(MoexSource(), settings=settings()) assert result.warnings == [ f"SBER: разрыв в истории {priced[1].isoformat()}..{priced[2].isoformat()}" ] def board(name: str, *, primary: bool, since: str | None, till: str | None) -> BoardInfo: return BoardInfo( secid="TBRU", board=name, market="shares", engine="stock", is_primary=primary, currency="SUR", history_from=date.fromisoformat(since) if since else None, history_till=date.fromisoformat(till) if till else None, ) MOVED_FROM = board("TQTF", primary=False, since="2021-07-12", till="2026-06-19") MOVED_TO = board("TQBR", primary=True, since="2026-06-22", till="2026-09-17") def test_a_remembered_board_that_stopped_trading_is_dropped(): """The June 2026 fund move: keeping TQTF freezes a fund on the day it left that board.""" assert choose_board([MOVED_FROM, MOVED_TO], "TQTF").board == "TQBR" def test_a_remembered_board_that_is_still_current_is_kept(): """A paper listed on several live boards keeps the board the instrument was resolved to.""" small = board("SMAL", primary=False, since="2011-11-21", till="2026-09-17") assert choose_board([small, MOVED_TO], "SMAL").board == "SMAL" def test_history_before_a_move_is_asked_of_the_board_the_paper_left(): since, until = date(2025, 7, 8), date(2026, 9, 18) assert history_legs([MOVED_FROM, MOVED_TO], MOVED_TO, since, until) == [ (MOVED_FROM, since, date(2026, 6, 21)), (MOVED_TO, date(2026, 6, 22), until), ] def test_a_board_that_covers_the_whole_window_is_asked_alone(): """SBER also trades on SMAL; pricing it from two boards at once would mix the two.""" since = date(2025, 1, 1) whole = board("TQBR", primary=True, since="2013-03-25", till="2026-09-17") assert history_legs([whole, MOVED_FROM], whole, since, date(2026, 9, 18)) == [ (whole, since, date(2026, 9, 18)) ] async def test_a_backfill_spans_both_sides_of_a_board_move(app, mock_http, run_sync): first_held = monday_back(120) move = TODAY - timedelta(days=30) instrument_id = await seed(first_held=first_held, priced=[], asked_from=None) history = mock_iss( mock_http, boards=[ [ "SBER", "TQTF", "shares", "stock", 0, "2021-07-12", (move - timedelta(days=3)).isoformat(), ], ["SBER", "TQBR", "shares", "stock", 1, move.isoformat(), TODAY.isoformat()], ], ) await run_sync(MoexSource(), settings=settings()) assert legs(history) == [ ("TQTF", first_held, move - timedelta(days=1)), ("TQBR", move, TODAY), ] count, low, high = await stored_days(instrument_id) assert low == first_held # the stretch on the old board is stored under the same instrument assert high >= TODAY - timedelta(days=2) # ... and the new board carries it to today assert count > 60