From d2df86ce3365b1e4aa23e015344dea495f07eb71 Mon Sep 17 00:00:00 2001 From: Dmitry Date: Sun, 20 Sep 2026 10:46:24 +0300 Subject: [PATCH] =?UTF-8?q?feat(moex):=20=D0=B8=D1=81=D1=82=D0=BE=D1=80?= =?UTF-8?q?=D0=B8=D1=8F=20=D0=B1=D1=83=D0=BC=D0=B0=D0=B3=20=D1=81=20=D0=BB?= =?UTF-8?q?=D0=B8=D1=81=D1=82=D0=B8=D0=BD=D0=B3=D0=B0=20=D0=B8=20=D1=86?= =?UTF-8?q?=D0=B5=D0=BD=D1=8B=20=D0=B8=D0=BD=D0=B4=D0=B5=D0=BA=D1=81=D0=BE?= =?UTF-8?q?=D0=B2=20=D0=B4=D0=BB=D1=8F=20=D0=B1=D0=B5=D0=BD=D1=87=D0=BC?= =?UTF-8?q?=D0=B0=D1=80=D0=BA=D0=BE=D0=B2?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Окно бэкфилла начинается с history_from доски, а не с первой покупки: график цены показывает историю бумаги целиком. Для активных бенчмарков source=moex синк создаёт instrument и качает индекс с его доски (IMOEX и RGBITR на SNDX, MCFTR на RTSI). Миграция засевает IMOEX, MCFTR и RGBITR; тесты чистят таблицы перед каждым тестом, чтобы сид не попадал в первый из них. --- AGENTS.md | 5 +- .../f3a91c7d5e28_seed_moex_benchmarks.py | 49 ++++++ backend/src/fintracker/sources/moex/sync.py | 140 +++++++++++++++-- backend/tests/conftest.py | 3 + backend/tests/sources/test_moex_sync.py | 143 +++++++++++++++++- 5 files changed, 317 insertions(+), 23 deletions(-) create mode 100644 backend/alembic/versions/f3a91c7d5e28_seed_moex_benchmarks.py diff --git a/AGENTS.md b/AGENTS.md index bcfe686..a49c4d1 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -165,7 +165,10 @@ just revision "msg" # новая alembic-миграция из изме - `analytics/benchmarks.py` — TWR индекса на сетке дат портфеля; `kind` (`price` vs `total_return`) выставляется наружу, а не скрывается: сравнение с ценовым IMOEX без дивидендов льстит портфелю на несколько % годовых, это осознанный выбор пользователя, - какой индекс сравнивать; + какой индекс сравнивать. Цены индексов тянет `sources/moex` (`_ensure_benchmark_instruments` + создаёт `instrument` для активной строки `benchmark`; доска берётся из ISS — MCFTR на RTSI, + IMOEX и RGBITR на SNDX); миграция `f3a91c7d5e28` засевает IMOEX, MCFTR (оба `is_default`) + и RGBITR. История бумаг и индексов качается с листинга, а не с первой покупки; - `analytics/goals.py` + `api/routers/goals.py` — прогресс цели и требуемый ежемесячный взнос по trailing XIRR; - четыре новых шага в `register_steps`: `benchmarks` после `returns` (общая сетка дат), diff --git a/backend/alembic/versions/f3a91c7d5e28_seed_moex_benchmarks.py b/backend/alembic/versions/f3a91c7d5e28_seed_moex_benchmarks.py new file mode 100644 index 0000000..8e7edb8 --- /dev/null +++ b/backend/alembic/versions/f3a91c7d5e28_seed_moex_benchmarks.py @@ -0,0 +1,49 @@ +"""benchmark: the MOEX indices the portfolio is compared against + +The list is data, but there is no screen to add to it, and an empty `benchmark` table leaves +`/analytics/benchmarks` and the overlay on the price chart with nothing to draw. IMOEX and +MCFTR are the pair the comparison is built around (a price index next to its dividend- +reinvested twin, so the gap between them is visible); RGBITR is there for a bond-heavy +portfolio and is not shown unasked. A row that is already there is left as the user set it. + +Revision ID: f3a91c7d5e28 +Revises: e8b21f6a90c3 +Create Date: 2026-09-20 18:00:00.000000 +""" + +from __future__ import annotations + +from collections.abc import Sequence + +import sqlalchemy as sa +from alembic import op + +revision: str = "f3a91c7d5e28" +down_revision: str | None = "e8b21f6a90c3" +branch_labels: str | Sequence[str] | None = None +depends_on: str | Sequence[str] | None = None + +BENCHMARKS = [ + ("IMOEX", "Индекс МосБиржи", "price", True), + ("MCFTR", "Индекс МосБиржи полной доходности «брутто»", "total_return", True), + ( + "RGBITR", + "Индекс МосБиржи государственных облигаций (полной доходности)", + "total_return", + False, + ), +] + + +def upgrade() -> None: + insert = sa.text( + "INSERT INTO benchmark (code, name, kind, source, currency, is_default, is_active) " + "VALUES (:code, :name, CAST(:kind AS benchmark_kind), 'moex', 'RUB', :is_default, true) " + "ON CONFLICT (code) DO NOTHING" + ) + for code, name, kind, is_default in BENCHMARKS: + op.execute(insert.bindparams(code=code, name=name, kind=kind, is_default=is_default)) + + +def downgrade() -> None: + op.execute(sa.text("DELETE FROM benchmark WHERE code IN ('IMOEX', 'MCFTR', 'RGBITR')")) diff --git a/backend/src/fintracker/sources/moex/sync.py b/backend/src/fintracker/sources/moex/sync.py index c3a56b3..32399bf 100644 --- a/backend/src/fintracker/sources/moex/sync.py +++ b/backend/src/fintracker/sources/moex/sync.py @@ -1,8 +1,12 @@ """The `moex` source: fill `price_daily` / `price_last` for papers the portfolio holds. Scope is derived from the ledger, not configured: only instruments that appear in `event` -are priced, and each is fetched from the first day it was held rather than from its -listing — pricing a paper for years before it was bought would be thousands of useless rows. +are priced, plus the indices behind the active MOEX benchmarks (`_ensure_benchmark_instruments` +gives each one an `instrument`; `analytics/benchmarks.py` and the chart overlay read them +from `price_daily` like any other paper). Each is fetched from its listing on the board it +is priced from (`history_start`), not from the day it was first held: the chart on the +instrument card shows the paper's own history, and the analytics never read a price from +before the first purchase, so the older rows cost only disk. Each instrument's board is resolved from `/securities/{secid}.json` and cached in `instrument.board`/`exchange`. A cached board is only kept while it is still trading, and the @@ -17,10 +21,10 @@ the past never closed — which is what left TWR skipping days for want of a pri window comes from the instrument's own state instead: * `price_coverage.history_from` — the earliest date we have already *asked* ISS for. While - it is later than the day the paper was first held (or missing), the run does a full sweep - from that day; once recorded, the paper falls back to the incremental window. Asking is - what gets remembered, not receiving: a stretch the exchange has nothing for would - otherwise be re-requested on every single run, forever. + it is later than the start of the paper's history (`history_start`) or missing, the run + does a full sweep from that day; once recorded, the paper falls back to the incremental + window. Asking is what gets remembered, not receiving: a stretch the exchange has nothing + for would otherwise be re-requested on every single run, forever. * `max(price_daily.d)` — the newest day stored. The incremental window starts a few days before it, because ISS revises a session's settlement price after the close, and because anchoring on the instrument's own data (rather than on the run date) makes a run that @@ -31,7 +35,7 @@ from __future__ import annotations import logging from collections.abc import Sequence -from dataclasses import dataclass +from dataclasses import dataclass, replace from datetime import UTC, date, datetime, timedelta from decimal import Decimal @@ -40,7 +44,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert from sqlalchemy.ext.asyncio import AsyncSession from fintracker.analytics import today_local -from fintracker.models import AssetClass, Event, EventStatus, Instrument +from fintracker.models import AssetClass, Benchmark, Event, EventStatus, Instrument from fintracker.models.pricing import PriceCoverage, PriceDaily, PriceLast from fintracker.sources.base import SyncContext, SyncResult from fintracker.sources.moex.client import BoardInfo, Candle, MoexClient, MoexError @@ -59,7 +63,17 @@ calendar artefact rather than a paper we failed to download. CHUNK = 500 #: Only these can be priced on MOEX; currencies come from the CBR and custom holdings by hand. -PRICEABLE = {AssetClass.share, AssetClass.bond, AssetClass.etf, AssetClass.fund} +PRICEABLE = { + AssetClass.share, + AssetClass.bond, + AssetClass.etf, + AssetClass.fund, + AssetClass.market_index, +} + +BENCHMARK_SINCE = date(1990, 1, 1) +"""An index has no first purchase: this stands in for it, and `history_start` pulls it forward +to the day the index began (IMOEX: 1997-09-22).""" @dataclass(frozen=True) @@ -73,6 +87,8 @@ class Target: board: str | None exchange: str | None asset_class: AssetClass + is_benchmark: bool = False + """Priced for a comparison, not because the portfolio holds it.""" @dataclass(frozen=True) @@ -89,6 +105,23 @@ class Coverage: """Breaks longer than GAP_DAYS between two stored days, as (last before, first after).""" +def history_start(first_held: date, board: BoardInfo) -> date: + """The day to start a paper's history from: its listing on the board it is priced from. + + The chart on the instrument card shows the paper's own price history, not just the stretch + it was held — a chart that begins on the day of the first purchase says nothing about + where the paper stood when it was bought. The board's `history_from` is the earliest day + ISS has anything for, so it bounds the sweep without guessing a floor. + + A board that started *after* the first purchase (the T-Bank funds that moved from TQTF to + TQBR in June 2026) does not go back that far: the day held stays the start, and + `history_legs` asks the board it traded on before the move for the rest. + """ + if board.history_from is None: + return first_held + return min(first_held, board.history_from) + + def needs_backfill(target: Target, coverage: Coverage) -> bool: """True while the paper's history has never been asked for from the day it was held.""" return coverage.history_from is None or coverage.history_from > target.since @@ -166,6 +199,7 @@ class MoexSource: async def sync(self, ctx: SyncContext) -> SyncResult: session = ctx.session today = today_local() + await _ensure_benchmark_instruments(session) targets = await _targets(session) if not targets: log.info("moex: no priceable instruments in the ledger yet") @@ -183,6 +217,8 @@ class MoexSource: continue board, boards = resolved coverage = coverages.get(target.instrument_id, Coverage()) + first_held = target.since + target = replace(target, since=history_start(first_held, board)) backfill = needs_backfill(target, coverage) since, until = fetch_window(target, coverage, today) candles: list[Candle] = [] @@ -207,14 +243,21 @@ class MoexSource: log.info("moex: %s backfilled from %s", target.secid, since) else: # a sweep closes these itself; reporting them otherwise keeps a paper that - # stopped trading from looking like a failed download, and vice versa - warnings += [ - f"{target.secid}: разрыв в истории {a.isoformat()}..{b.isoformat()}" - for a, b in coverage.gaps - ] + # stopped trading from looking like a failed download, and vice versa. + # Only breaks in the stretch the paper was held: the older history is there + # for the chart, and a halt years before the purchase is not ours to fix. + # An index was never held — the comparison reports its own `days_skipped`. + if not target.is_benchmark: + warnings += [ + f"{target.secid}: разрыв в истории {a.isoformat()}..{b.isoformat()}" + for a, b in coverage.gaps + if a >= first_held + ] # remembered even when ISS returned nothing: we asked, and asking is the state await _store_coverage(session, target, since) + if target.is_benchmark: + continue # an index has no live quote to hold: only its daily close is read last = await moex.last_price( board.secid, engine=board.engine, market=board.market, board=board.board ) @@ -264,7 +307,7 @@ async def _targets(session: AsyncSession) -> list[Target]: ) ) ).all() - return [ + targets = [ Target( instrument_id=r.id, secid=r.ticker, @@ -278,6 +321,73 @@ async def _targets(session: AsyncSession) -> list[Target]: if r.asset_class in PRICEABLE and r.first_held ] + held = {t.instrument_id for t in targets} + indices = ( + await session.execute( + select(Instrument) + .join(Benchmark, Benchmark.instrument_id == Instrument.id) + .where(Benchmark.source == SOURCE, Benchmark.is_active, Instrument.ticker.is_not(None)) + ) + ).scalars() + return targets + [ + Target( + instrument_id=i.id, + secid=i.ticker or "", + since=BENCHMARK_SINCE, + nominal=None, + board=i.board, + exchange=i.exchange, + asset_class=i.asset_class, + is_benchmark=True, + ) + for i in indices + if i.id not in held + ] + + +async def _ensure_benchmark_instruments(session: AsyncSession) -> None: + """Give every active MOEX benchmark the `instrument` its history is stored under. + + The benchmark list is the user's data, and a row added through the API has no instrument + yet (`instrument_id` is NULL until the index is first synced). The board is left empty: + indices do not share one — IMOEX and RGBITR are on SNDX, MCFTR is on RTSI — so `_board_for` + resolves it from ISS like it does for any paper. + """ + pending = ( + ( + await session.execute( + select(Benchmark).where( + Benchmark.source == SOURCE, + Benchmark.is_active, + Benchmark.instrument_id.is_(None), + ) + ) + ) + .scalars() + .all() + ) + for benchmark in pending: + instrument = ( + await session.execute( + select(Instrument).where( + Instrument.ticker == benchmark.code, + Instrument.asset_class == AssetClass.market_index, + ) + ) + ).scalar_one_or_none() + if instrument is None: + instrument = Instrument( + asset_class=AssetClass.market_index, + ticker=benchmark.code, + name=benchmark.name, + currency=benchmark.currency, + ) + session.add(instrument) + await session.flush() + benchmark.instrument_id = instrument.id + if pending: + await session.flush() + async def _coverage(session: AsyncSession) -> dict[int, Coverage]: """Stored span, remembered backfill depth and long breaks, for every instrument at once.""" diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index a0bf9b9..7a8b08c 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -56,6 +56,9 @@ async def app(migrated: str): from fintracker.db import reset_engine auth_router._login_limiter = None # fresh rate limiter per test + # rows a migration seeds (the default benchmarks) are in the first test's database only, + # since the tables are emptied after each test — start every test from the same blank one + await _truncate_all() application = app_module.create_app() yield application await _truncate_all() diff --git a/backend/tests/sources/test_moex_sync.py b/backend/tests/sources/test_moex_sync.py index f1ce0f6..1c940bf 100644 --- a/backend/tests/sources/test_moex_sync.py +++ b/backend/tests/sources/test_moex_sync.py @@ -18,7 +18,15 @@ from factories import make_account, make_event, make_instrument, make_price from fintracker.analytics import today_local from fintracker.config import Settings from fintracker.db import get_sessionmaker -from fintracker.models import EventKind, PriceCoverage, PriceDaily +from fintracker.models import ( + AssetClass, + Benchmark, + BenchmarkKind, + EventKind, + Instrument, + PriceCoverage, + PriceDaily, +) from fintracker.sources.moex.client import BoardInfo from fintracker.sources.moex.sync import ( OVERLAP_DAYS, @@ -28,11 +36,14 @@ from fintracker.sources.moex.sync import ( choose_board, fetch_window, history_legs, + history_start, needs_backfill, ) ISS = "https://iss.moex.com/iss" TODAY = today_local() +LISTING = date(2013, 3, 25) +"""When the mocked TQBR started: the day a backfill now reaches back to.""" def monday_back(days: int) -> date: @@ -198,8 +209,27 @@ def test_a_paper_without_any_price_is_asked_from_the_day_it_was_held(): assert fetch_window(target(since), Coverage(), TODAY) == (since, TODAY) -async def test_a_hole_in_the_past_is_pulled_back_to_the_first_held_day(app, mock_http, run_sync): - """The TBRU@ case: held since spring, priced only from the day the source first saw it.""" +def test_history_starts_at_the_listing_when_that_is_before_the_first_purchase(): + whole = board("TQBR", primary=True, since="2013-03-25", till="2026-09-17") + assert history_start(date(2025, 7, 8), whole) == date(2013, 3, 25) + + +def test_history_starts_at_the_first_purchase_when_the_board_began_later(): + """A board the paper moved to has no history before the move — the old board supplies it.""" + assert history_start(date(2025, 7, 8), MOVED_TO) == date(2025, 7, 8) + + +def test_history_starts_at_the_first_purchase_when_the_board_publishes_no_range(): + unknown = board("TQBR", primary=True, since=None, till=None) + assert history_start(date(2025, 7, 8), unknown) == date(2025, 7, 8) + + +async def test_a_hole_in_the_past_is_pulled_back_to_the_listing(app, mock_http, run_sync): + """The TBRU@ case: held since spring, priced only from the day the source first saw it. + + The sweep goes back past the first purchase to the listing, so the chart on the + instrument card has the paper's own history rather than only the stretch it was held. + """ first_held = monday_back(60) recent = [TODAY - timedelta(days=n) for n in (3, 2, 1)] instrument_id = await seed(first_held=first_held, priced=recent, asked_from=recent[0]) @@ -207,9 +237,9 @@ async def test_a_hole_in_the_past_is_pulled_back_to_the_first_held_day(app, mock history = mock_iss(mock_http) result = await run_sync(MoexSource(), settings=settings()) - assert windows(history) == [(first_held, TODAY)] + assert windows(history) == [(LISTING, TODAY)] count, low, high = await stored_days(instrument_id) - assert low == first_held + assert low == LISTING assert high >= recent[-1] assert count > len(recent) assert result.counts["backfilled"] == 1 @@ -223,7 +253,7 @@ async def test_a_complete_history_only_re_reads_the_overlap_window(app, mock_htt for n in range(61) if (first_held + timedelta(days=n)).isoweekday() < 6 ] - await seed(first_held=first_held, priced=priced, asked_from=first_held) + await seed(first_held=first_held, priced=priced, asked_from=LISTING) history = mock_iss(mock_http) result = await run_sync(MoexSource(), settings=settings()) @@ -256,7 +286,7 @@ async def test_a_long_break_inside_a_settled_history_is_reported(app, mock_http, so the gap is surfaced as a warning instead of being silently re-fetched every run.""" first_held = monday_back(120) priced = [first_held, first_held + timedelta(days=1), TODAY - timedelta(days=1)] - await seed(first_held=first_held, priced=priced, asked_from=first_held) + await seed(first_held=first_held, priced=priced, asked_from=LISTING) mock_iss(mock_http) result = await run_sync(MoexSource(), settings=settings()) @@ -266,6 +296,22 @@ async def test_a_long_break_inside_a_settled_history_is_reported(app, mock_http, ] +async def test_a_break_before_the_first_purchase_is_not_reported(app, mock_http, run_sync): + """The older history exists for the chart; a halt years before the purchase is not ours.""" + first_held = monday_back(60) + held = [ + first_held + timedelta(days=n) + for n in range(61) + if (first_held + timedelta(days=n)).isoweekday() < 6 + ] + await seed(first_held=first_held, priced=[LISTING, *held], asked_from=LISTING) + + mock_iss(mock_http) + result = await run_sync(MoexSource(), settings=settings()) + + assert result.warnings == [] + + def board(name: str, *, primary: bool, since: str | None, till: str | None) -> BoardInfo: return BoardInfo( secid="TBRU", @@ -343,3 +389,86 @@ async def test_a_backfill_spans_both_sides_of_a_board_move(app, mock_http, run_s assert low == first_held # the stretch on the old board is stored under the same instrument assert high >= TODAY - timedelta(days=2) # ... and the new board carries it to today assert count > 60 + + +async def add_benchmark(code: str, *, source: str = "moex", active: bool = True) -> int: + async with get_sessionmaker()() as session: + benchmark = Benchmark( + code=code, + name=f"Индекс {code}", + kind=BenchmarkKind.total_return, + source=source, + currency="RUB", + is_default=False, + is_active=active, + ) + session.add(benchmark) + await session.commit() + return benchmark.id + + +async def benchmark_instrument(benchmark_id: int) -> Instrument | None: + async with get_sessionmaker()() as session: + benchmark = await session.get(Benchmark, benchmark_id) + assert benchmark is not None + if benchmark.instrument_id is None: + return None + return await session.get(Instrument, benchmark.instrument_id) + + +async def test_an_active_benchmark_gets_an_instrument_and_its_whole_history( + app, mock_http, run_sync +): + """MCFTR is on RTSI, not SNDX with the other indices — the board comes from ISS.""" + benchmark_id = await add_benchmark("MCFTR") + listed = date(2003, 2, 26) + history = mock_iss( + mock_http, + boards=[["MCFTR", "RTSI", "index", "stock", 1, listed.isoformat(), TODAY.isoformat()]], + ) + + result = await run_sync(MoexSource(), settings=settings()) + + instrument = await benchmark_instrument(benchmark_id) + assert instrument is not None + assert instrument.asset_class == AssetClass.market_index + assert (instrument.ticker, instrument.board, instrument.exchange) == ( + "MCFTR", + "RTSI", + "index", + ) + assert legs(history) == [("RTSI", listed, TODAY)] + count, low, high = await stored_days(instrument.id) + assert low == listed + assert high >= TODAY - timedelta(days=3) + assert count > 1000 + assert result.counts["last"] == 0 # an index has no live quote to keep + assert result.warnings == [] + + +async def test_a_second_run_does_not_duplicate_the_benchmark_instrument(app, mock_http, run_sync): + benchmark_id = await add_benchmark("IMOEX") + mock_iss( + mock_http, + boards=[["IMOEX", "SNDX", "index", "stock", 1, "1997-09-22", TODAY.isoformat()]], + ) + + await run_sync(MoexSource(), settings=settings()) + first = await benchmark_instrument(benchmark_id) + second_run = await run_sync(MoexSource(), settings=settings()) + second = await benchmark_instrument(benchmark_id) + + assert first is not None and second is not None + assert first.id == second.id + assert second_run.counts["backfilled"] == 0 + + +async def test_an_inactive_or_manual_benchmark_is_left_alone(app, mock_http, run_sync): + inactive = await add_benchmark("RGBITR", active=False) + manual = await add_benchmark("SPX", source="manual") + + result = await run_sync(MoexSource(), settings=settings()) # no ISS route: any call fails + + assert await benchmark_instrument(inactive) is None + assert await benchmark_instrument(manual) is None + assert result.counts == {"prices": 0}