ada/fin-tracker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(moex): история бумаг с листинга и цены индексов для бенчмарков

Browse Source 
Окно бэкфилла начинается с history_from доски, а не с первой покупки: график цены показывает историю бумаги целиком. Для активных бенчмарков source=moex синк создаёт instrument и качает индекс с его доски (IMOEX и RGBITR на SNDX, MCFTR на RTSI). Миграция засевает IMOEX, MCFTR и RGBITR; тесты чистят таблицы перед каждым тестом, чтобы сид не попадал в первый из них.
This commit is contained in:
Dmitry
2026-09-20 10:46:24 +03:00
parent 05affeea29
commit d2df86ce33
5 changed files with 317 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/src/fintracker/sources/moex/sync.py
+125 -15
View File
@@ -1,8 +1,12 @@
"""The `moex` source: fill `price_daily` / `price_last` for papers the portfolio holds.
Scope is derived from the ledger, not configured: only instruments that appear in `event`
are priced, and each is fetched from the first day it was held rather than from its
listing — pricing a paper for years before it was bought would be thousands of useless rows.
are priced, plus the indices behind the active MOEX benchmarks (`_ensure_benchmark_instruments`
gives each one an `instrument`; `analytics/benchmarks.py` and the chart overlay read them
from `price_daily` like any other paper). Each is fetched from its listing on the board it
is priced from (`history_start`), not from the day it was first held: the chart on the
instrument card shows the paper's own history, and the analytics never read a price from
before the first purchase, so the older rows cost only disk.
Each instrument's board is resolved from `/securities/{secid}.json` and cached in
`instrument.board`/`exchange`. A cached board is only kept while it is still trading, and the
@@ -17,10 +21,10 @@ the past never closed — which is what left TWR skipping days for want of a pri
window comes from the instrument's own state instead:
* `price_coverage.history_from` — the earliest date we have already *asked* ISS for. While
it is later than the day the paper was first held (or missing), the run does a full sweep
from that day; once recorded, the paper falls back to the incremental window. Asking is
what gets remembered, not receiving: a stretch the exchange has nothing for would
otherwise be re-requested on every single run, forever.
it is later than the start of the paper's history (`history_start`) or missing, the run
does a full sweep from that day; once recorded, the paper falls back to the incremental
window. Asking is what gets remembered, not receiving: a stretch the exchange has nothing
for would otherwise be re-requested on every single run, forever.
* `max(price_daily.d)` — the newest day stored. The incremental window starts a few days
before it, because ISS revises a session's settlement price after the close, and because
anchoring on the instrument's own data (rather than on the run date) makes a run that
@@ -31,7 +35,7 @@ from __future__ import annotations
import logging
from collections.abc import Sequence
from dataclasses import dataclass
from dataclasses import dataclass, replace
from datetime import UTC, date, datetime, timedelta
from decimal import Decimal
@@ -40,7 +44,7 @@ from sqlalchemy.dialects.postgresql import insert as pg_insert
from sqlalchemy.ext.asyncio import AsyncSession
from fintracker.analytics import today_local
from fintracker.models import AssetClass, Event, EventStatus, Instrument
from fintracker.models import AssetClass, Benchmark, Event, EventStatus, Instrument
from fintracker.models.pricing import PriceCoverage, PriceDaily, PriceLast
from fintracker.sources.base import SyncContext, SyncResult
from fintracker.sources.moex.client import BoardInfo, Candle, MoexClient, MoexError
@@ -59,7 +63,17 @@ calendar artefact rather than a paper we failed to download.
CHUNK = 500
#: Only these can be priced on MOEX; currencies come from the CBR and custom holdings by hand.
PRICEABLE = {AssetClass.share, AssetClass.bond, AssetClass.etf, AssetClass.fund}
PRICEABLE = {
AssetClass.share,
AssetClass.bond,
AssetClass.etf,
AssetClass.fund,
AssetClass.market_index,
}
BENCHMARK_SINCE = date(1990, 1, 1)
"""An index has no first purchase: this stands in for it, and `history_start` pulls it forward
to the day the index began (IMOEX: 1997-09-22)."""
@dataclass(frozen=True)
@@ -73,6 +87,8 @@ class Target:
board: str | None
exchange: str | None
asset_class: AssetClass
is_benchmark: bool = False
"""Priced for a comparison, not because the portfolio holds it."""
@dataclass(frozen=True)
@@ -89,6 +105,23 @@ class Coverage:
"""Breaks longer than GAP_DAYS between two stored days, as (last before, first after)."""
def history_start(first_held: date, board: BoardInfo) -> date:
"""The day to start a paper's history from: its listing on the board it is priced from.
The chart on the instrument card shows the paper's own price history, not just the stretch
it was held — a chart that begins on the day of the first purchase says nothing about
where the paper stood when it was bought. The board's `history_from` is the earliest day
ISS has anything for, so it bounds the sweep without guessing a floor.
A board that started *after* the first purchase (the T-Bank funds that moved from TQTF to
TQBR in June 2026) does not go back that far: the day held stays the start, and
`history_legs` asks the board it traded on before the move for the rest.
"""
if board.history_from is None:
return first_held
return min(first_held, board.history_from)
def needs_backfill(target: Target, coverage: Coverage) -> bool:
"""True while the paper's history has never been asked for from the day it was held."""
return coverage.history_from is None or coverage.history_from > target.since
@@ -166,6 +199,7 @@ class MoexSource:
async def sync(self, ctx: SyncContext) -> SyncResult:
session = ctx.session
today = today_local()
await _ensure_benchmark_instruments(session)
targets = await _targets(session)
if not targets:
log.info("moex: no priceable instruments in the ledger yet")
@@ -183,6 +217,8 @@ class MoexSource:
continue
board, boards = resolved
coverage = coverages.get(target.instrument_id, Coverage())
first_held = target.since
target = replace(target, since=history_start(first_held, board))
backfill = needs_backfill(target, coverage)
since, until = fetch_window(target, coverage, today)
candles: list[Candle] = []
@@ -207,14 +243,21 @@ class MoexSource:
log.info("moex: %s backfilled from %s", target.secid, since)
else:
# a sweep closes these itself; reporting them otherwise keeps a paper that
# stopped trading from looking like a failed download, and vice versa
warnings += [
f"{target.secid}: разрыв в истории {a.isoformat()}..{b.isoformat()}"
for a, b in coverage.gaps
]
# stopped trading from looking like a failed download, and vice versa.
# Only breaks in the stretch the paper was held: the older history is there
# for the chart, and a halt years before the purchase is not ours to fix.
# An index was never held — the comparison reports its own `days_skipped`.
if not target.is_benchmark:
warnings += [
f"{target.secid}: разрыв в истории {a.isoformat()}..{b.isoformat()}"
for a, b in coverage.gaps
if a >= first_held
]
# remembered even when ISS returned nothing: we asked, and asking is the state
await _store_coverage(session, target, since)
if target.is_benchmark:
continue # an index has no live quote to hold: only its daily close is read
last = await moex.last_price(
board.secid, engine=board.engine, market=board.market, board=board.board
)
@@ -264,7 +307,7 @@ async def _targets(session: AsyncSession) -> list[Target]:
)
)
).all()
return [
targets = [
Target(
instrument_id=r.id,
secid=r.ticker,
@@ -278,6 +321,73 @@ async def _targets(session: AsyncSession) -> list[Target]:
if r.asset_class in PRICEABLE and r.first_held
]
held = {t.instrument_id for t in targets}
indices = (
await session.execute(
select(Instrument)
.join(Benchmark, Benchmark.instrument_id == Instrument.id)
.where(Benchmark.source == SOURCE, Benchmark.is_active, Instrument.ticker.is_not(None))
)
).scalars()
return targets + [
Target(
instrument_id=i.id,
secid=i.ticker or "",
since=BENCHMARK_SINCE,
nominal=None,
board=i.board,
exchange=i.exchange,
asset_class=i.asset_class,
is_benchmark=True,
)
for i in indices
if i.id not in held
]
async def _ensure_benchmark_instruments(session: AsyncSession) -> None:
"""Give every active MOEX benchmark the `instrument` its history is stored under.
The benchmark list is the user's data, and a row added through the API has no instrument
yet (`instrument_id` is NULL until the index is first synced). The board is left empty:
indices do not share one — IMOEX and RGBITR are on SNDX, MCFTR is on RTSI — so `_board_for`
resolves it from ISS like it does for any paper.
"""
pending = (
(
await session.execute(
select(Benchmark).where(
Benchmark.source == SOURCE,
Benchmark.is_active,
Benchmark.instrument_id.is_(None),
)
)
)
.scalars()
.all()
)
for benchmark in pending:
instrument = (
await session.execute(
select(Instrument).where(
Instrument.ticker == benchmark.code,
Instrument.asset_class == AssetClass.market_index,
)
)
).scalar_one_or_none()
if instrument is None:
instrument = Instrument(
asset_class=AssetClass.market_index,
ticker=benchmark.code,
name=benchmark.name,
currency=benchmark.currency,
)
session.add(instrument)
await session.flush()
benchmark.instrument_id = instrument.id
if pending:
await session.flush()
async def _coverage(session: AsyncSession) -> dict[int, Coverage]:
"""Stored span, remembered backfill depth and long breaks, for every instrument at once."""