feat(tinvest): сектор инструмента из потиповых справочников

GetInstrumentBy отдаёт общую запись, в которой поля sector нет вовсе — из-за этого сектор не был заполнен ни у одного инструмента, и аллокация по нему показывала один бакет unknown на все двадцать. Сектор живёт только в Shares/Bonds/Etfs, поэтому резолв добивает недостающие поля из потипового справочника: по uid, затем figi, затем ticker+board. Логика разрешения uid не тронута — одна бумага приезжает под несколькими instrument_uid, и InstrumentAlias с _match_by_identity остаются единственным местом, которое это разбирает. Справочник тянется лениво и один раз на экземпляр клиента, и только для встреченных типов: это 2000-5000 записей на тип и около 14 секунд на все три. _backfill_sectors нужен отдельно от резолва: у уже известных бумаг uid не попадает в missing, и без отдельного прохода они никогда бы не переразрешились. Пустой сектор остаётся NULL, строка "unknown" не пишется никогда — это литерал самой аллокации, и данные, притворяющиеся ответом, отличить от отсутствия ответа нельзя. Из 80 инструментов сектор заполнен у 73. Остальные семь честно пустые: валюта, пять фондов, у которых поле пусто у самого T-Invest, и внебиржевой структурный продукт.
2026-09-18 15:07:18 +03:00
parent 1ceeba2c48
commit 582b859e56
4 changed files with 398 additions and 9 deletions
@@ -172,9 +172,9 @@ async def rebuild_allocation(session: AsyncSession) -> None:
 def _report_unknown(holdings: Sequence[Holding]) -> None:
    """A dimension that is mostly `unknown` is a missing attribute, not a portfolio shape.

-    Sector is the live case: the instrument sync reads `GetInstrumentBy`, whose response has
-    no sector field at all — it would take the per-type Shares/Bonds calls. Until then the
-    chart is honestly empty, and says so here instead of looking broken.
+    What is left after `sources/tinvest` learned to read the per-type Shares/Bonds listings
+    is the genuinely sectorless rest — currencies, and papers the listing does not carry —
+    so the chart says how much of itself it cannot explain instead of looking broken.
    """
    if not holdings:
        return
@@ -17,8 +17,9 @@ from __future__ import annotations

 import asyncio
 import logging
+import re
 from collections.abc import AsyncIterator, Callable, Coroutine
-from dataclasses import dataclass, field, is_dataclass
+from dataclasses import dataclass, field, is_dataclass, replace
 from datetime import UTC, date, datetime
 from decimal import Decimal
 from enum import Enum
@@ -42,6 +43,12 @@ NANO = Decimal(10) ** 9
 MAX_RETRIES = 5
 DEFAULT_RESET_SECONDS = 5.0
 PAGE_LIMIT = 1000
+SECTOR_MAX_LEN = 64
+"""Width of `instrument.sector`; a longer value is a surprise, not something to truncate into."""
+
+DIRECTORY_KINDS = frozenset({"share", "bond", "etf"})
+"""Kinds whose per-type listing states a sector. Currencies have none by nature, and
+`GetInstrumentBy` already carries everything else we store about them."""

 T = TypeVar("T")

@@ -198,6 +205,8 @@ class TinvestClient:
        self._app_name = app_name
        self._client: Any = None
        self._ctx: Any = None
+        self._directories: dict[str, dict[str, InstrumentInfo]] = {}
+        """Per-kind reference listings, indexed by identity and fetched at most once a run."""

    async def __aenter__(self) -> TinvestClient:
        self._ctx = AsyncClient(self._token, app_name=self._app_name)
@@ -318,12 +327,16 @@ class TinvestClient:

        Keyed by the uid we ASKED for: the API may answer with a different `uid` for the
        same paper, and callers need to match on the id their own data carries.
+
+        `GetInstrumentBy` answers with a generic record that has no `sector` at all — that
+        field lives only in the per-type listings — so each resolved instrument is topped up
+        from the listing for its kind before it is handed back.
        """
        out: dict[str, InstrumentInfo] = {}
        for uid in sorted(uids):
            info = await self._instrument_by_uid(uid)
            if info is not None:
-                out[uid] = info
+                out[uid] = await self._with_reference_fields(info)
        return out

    async def _instrument_by_uid(self, uid: str) -> InstrumentInfo | None:
@@ -357,6 +370,68 @@ class TinvestClient:
        )
        return [_instrument(i, kind=kind) for i in resp.instruments]

+    async def reference_instrument(
+        self,
+        kind: str,
+        *,
+        uid: str | None = None,
+        figi: str | None = None,
+        ticker: str | None = None,
+        class_code: str | None = None,
+    ) -> InstrumentInfo | None:
+        """One instrument as the per-type listing describes it, or None if it is not there.
+
+        Public because instruments imported before sectors were read are never re-resolved
+        by uid — `sync.py` backfills them through this. The listing is fetched once per kind
+        per client, so a backfill over hundreds of papers still costs one RPC per kind.
+        """
+        if kind not in DIRECTORY_KINDS:
+            return None
+        index = await self._directory(kind)
+        for key in _identity_keys(uid=uid, figi=figi, ticker=ticker, class_code=class_code):
+            found = index.get(key)
+            if found is not None:
+                return found
+        return None
+
+    async def _directory(self, kind: str) -> dict[str, InstrumentInfo]:
+        """The per-type listing indexed by every identity it can be matched on."""
+        cached = self._directories.get(kind)
+        if cached is not None:
+            return cached
+        index: dict[str, InstrumentInfo] = {}
+        for info in await self.all_instruments(kind):
+            for key in _identity_keys(
+                uid=info.uid, figi=info.figi, ticker=info.ticker, class_code=info.class_code
+            ):
+                # first writer wins: a ticker reused on another board must not overwrite the
+                # uid/figi entries, which are the identities that cannot collide
+                index.setdefault(key, info)
+        self._directories[kind] = index
+        return index
+
+    async def _with_reference_fields(self, info: InstrumentInfo) -> InstrumentInfo:
+        """Fill the fields only the per-type listing states (sector, country of risk)."""
+        if info.sector is not None and info.country is not None:
+            return info
+        match = await self.reference_instrument(
+            info.kind,
+            uid=info.uid,
+            figi=info.figi,
+            ticker=info.ticker,
+            class_code=info.class_code,
+        )
+        if match is None:
+            # a delisted or non-exchange paper is simply absent from the listing; it keeps
+            # sector = NULL, which allocation reports as an honest gap
+            log.info("tinvest: %s (%s) is not in the %s listing", info.ticker, info.uid, info.kind)
+            return info
+        return replace(
+            info,
+            sector=info.sector or match.sector,
+            country=info.country or match.country,
+        )
+

 def _silence_sdk_telemetry() -> None:
    """Stop the SDK from reporting our errors to T-Bank's Sentry.
@@ -424,13 +499,52 @@ def _instrument(raw: Any, *, kind: str | None = None) -> InstrumentInfo:
        nominal=_money(nominal),
        nominal_currency=_currency(nominal),
        maturity_date=getattr(raw, "maturity_date", None),
-        sector=getattr(raw, "sector", None) or None,
+        sector=_sector(getattr(raw, "sector", None)),
        country=getattr(raw, "country_of_risk", None) or None,
        exchange=getattr(raw, "exchange", None) or None,
        payload=_as_dict(raw),
    )


+def _sector(value: Any) -> str | None:
+    """A sector as a stable key: lowercase, non-alphanumerics folded into underscores.
+
+    The value ends up in `instrument.sector` and from there in `metric_allocation.bucket`,
+    which the client labels by key — so the API's casing ("Financial", "health_care",
+    "Consumer Staples") has to be settled once, here, or the same sector splits into
+    several buckets. An absent sector stays NULL: `unknown` is allocation's own bucket for
+    a missing attribute, and storing that string would make a stated sector and a missing
+    one indistinguishable.
+    """
+    if value is None:
+        return None
+    key = re.sub(r"[^a-z0-9]+", "_", str(value).strip().lower()).strip("_")
+    return key[:SECTOR_MAX_LEN] or None
+
+
+def _identity_keys(
+    *,
+    uid: str | None = None,
+    figi: str | None = None,
+    ticker: str | None = None,
+    class_code: str | None = None,
+) -> list[str]:
+    """Namespaced lookup keys for one paper, strongest identity first.
+
+    The listing and `GetInstrumentBy` do not always agree on the uid of the same paper (a
+    fund's own uid vs the traded line's — see `sync.py`), so figi and ticker+board are kept
+    as fallbacks. Keys are namespaced because a figi and a ticker could otherwise collide.
+    """
+    keys = []
+    if uid:
+        keys.append(f"uid:{uid}")
+    if figi:
+        keys.append(f"figi:{figi}")
+    if ticker and class_code:
+        keys.append(f"ticker:{ticker}:{class_code}")
+    return keys
+
+
 def _kind_of(raw: Any) -> str:
    """Asset kind, preferring what the API states over guessing from the record's shape.

@@ -72,6 +72,14 @@ ASSET_CLASSES = {
    "currency": AssetClass.currency,
 }

+#: Asset classes whose T-Invest listing states a sector, and the listing that states it.
+#: Currencies have no sector by nature, so they are not worth a lookup.
+SECTOR_KINDS = {
+    AssetClass.share: "share",
+    AssetClass.bond: "bond",
+    AssetClass.etf: "etf",
+}
+
 #: T-Invest account types that are not really brokerage accounts we want in the ledger.
 SKIP_ACCOUNT_TYPES = frozenset({"ACCOUNT_TYPE_UNSPECIFIED"})

@@ -89,7 +97,14 @@ class TinvestSource:

        session = ctx.session
        cursors = _parse_cursor(ctx.cursor_before)
-        counts = {"accounts": 0, "operations": 0, "events": 0, "instruments": 0, "snapshots": 0}
+        counts = {
+            "accounts": 0,
+            "operations": 0,
+            "events": 0,
+            "instruments": 0,
+            "sectors": 0,
+            "snapshots": 0,
+        }
        warnings: list[str] = []
        new_cursors: dict[str, str] = dict(cursors)

@@ -127,6 +142,8 @@ class TinvestSource:
                        session, account_ids[info.id], snapshot
                    )

+            counts["sectors"] = await _backfill_sectors(session, client)
+
        await session.commit()
        log.info(
            "tinvest: %s accounts, %s operations, %s events, %s new instruments",
@@ -139,7 +156,7 @@ class TinvestSource:
            cursor_after=json.dumps(new_cursors, sort_keys=True),
            counts=counts,
            warnings=warnings,
-            changed=counts["events"] > 0 or counts["operations"] > 0,
+            changed=counts["events"] > 0 or counts["operations"] > 0 or counts["sectors"] > 0,
        )


@@ -288,9 +305,12 @@ async def _resolve_instruments(
            session.add(instrument)
            created += 1
        else:
-            # an instrument already known from another source gains its T-Invest identity
+            # an instrument already known from another source gains its T-Invest identity,
+            # and the attributes only T-Invest states (MOEX gives us neither)
            instrument.tinvest_uid = instrument.tinvest_uid or info.uid
            instrument.figi = instrument.figi or info.figi
+            instrument.sector = instrument.sector or info.sector
+            instrument.country = instrument.country or info.country
        await session.flush()
        known[asked_uid] = instrument.id
        known[info.uid] = instrument.id
@@ -312,6 +332,48 @@ async def _resolve_instruments(
    return {"by_uid": known, "created": created}


+async def _backfill_sectors(session: AsyncSession, client: TinvestClient) -> int:
+    """Fill sector (and country) on instruments that were resolved without them.
+
+    Uid resolution above never revisits an instrument we already know, so a paper imported
+    before the per-type listing was read would otherwise keep `sector = NULL` forever. The
+    listing is fetched at most once per kind per run, so this costs one RPC per kind no
+    matter how many instruments are missing. A paper the listing does not carry — delisted,
+    or traded outside the exchange — stays NULL rather than being guessed at; it is simply
+    re-checked on the next run.
+    """
+    instruments = (
+        (
+            await session.execute(
+                select(Instrument).where(
+                    Instrument.tinvest_uid.is_not(None),
+                    Instrument.sector.is_(None),
+                    Instrument.asset_class.in_(list(SECTOR_KINDS)),
+                )
+            )
+        )
+        .scalars()
+        .all()
+    )
+    filled = 0
+    for instrument in instruments:
+        match = await client.reference_instrument(
+            SECTOR_KINDS[instrument.asset_class],
+            uid=instrument.tinvest_uid,
+            figi=instrument.figi,
+            ticker=instrument.ticker,
+            class_code=instrument.board,
+        )
+        if match is None or match.sector is None:
+            continue
+        instrument.sector = match.sector
+        instrument.country = instrument.country or match.country
+        filled += 1
+    if instruments:
+        log.info("tinvest: sector filled for %d of %d instruments", filled, len(instruments))
+    return filled
+
+
 def _identities(operations: list[Operation]) -> dict[str, tuple[str | None, str | None]]:
    """instrument_uid -> (figi, position_uid), as the operations themselves report it."""
    out: dict[str, tuple[str | None, str | None]] = {}