feat(tinvest): сектор инструмента из потиповых справочников
GetInstrumentBy отдаёт общую запись, в которой поля sector нет вовсе — из-за этого сектор не был заполнен ни у одного инструмента, и аллокация по нему показывала один бакет unknown на все двадцать. Сектор живёт только в Shares/Bonds/Etfs, поэтому резолв добивает недостающие поля из потипового справочника: по uid, затем figi, затем ticker+board. Логика разрешения uid не тронута — одна бумага приезжает под несколькими instrument_uid, и InstrumentAlias с _match_by_identity остаются единственным местом, которое это разбирает. Справочник тянется лениво и один раз на экземпляр клиента, и только для встреченных типов: это 2000-5000 записей на тип и около 14 секунд на все три. _backfill_sectors нужен отдельно от резолва: у уже известных бумаг uid не попадает в missing, и без отдельного прохода они никогда бы не переразрешились. Пустой сектор остаётся NULL, строка "unknown" не пишется никогда — это литерал самой аллокации, и данные, притворяющиеся ответом, отличить от отсутствия ответа нельзя. Из 80 инструментов сектор заполнен у 73. Остальные семь честно пустые: валюта, пять фондов, у которых поле пусто у самого T-Invest, и внебиржевой структурный продукт.
This commit is contained in:
@@ -72,6 +72,14 @@ ASSET_CLASSES = {
|
||||
"currency": AssetClass.currency,
|
||||
}
|
||||
|
||||
#: Asset classes whose T-Invest listing states a sector, and the listing that states it.
|
||||
#: Currencies have no sector by nature, so they are not worth a lookup.
|
||||
SECTOR_KINDS = {
|
||||
AssetClass.share: "share",
|
||||
AssetClass.bond: "bond",
|
||||
AssetClass.etf: "etf",
|
||||
}
|
||||
|
||||
#: T-Invest account types that are not really brokerage accounts we want in the ledger.
|
||||
SKIP_ACCOUNT_TYPES = frozenset({"ACCOUNT_TYPE_UNSPECIFIED"})
|
||||
|
||||
@@ -89,7 +97,14 @@ class TinvestSource:
|
||||
|
||||
session = ctx.session
|
||||
cursors = _parse_cursor(ctx.cursor_before)
|
||||
counts = {"accounts": 0, "operations": 0, "events": 0, "instruments": 0, "snapshots": 0}
|
||||
counts = {
|
||||
"accounts": 0,
|
||||
"operations": 0,
|
||||
"events": 0,
|
||||
"instruments": 0,
|
||||
"sectors": 0,
|
||||
"snapshots": 0,
|
||||
}
|
||||
warnings: list[str] = []
|
||||
new_cursors: dict[str, str] = dict(cursors)
|
||||
|
||||
@@ -127,6 +142,8 @@ class TinvestSource:
|
||||
session, account_ids[info.id], snapshot
|
||||
)
|
||||
|
||||
counts["sectors"] = await _backfill_sectors(session, client)
|
||||
|
||||
await session.commit()
|
||||
log.info(
|
||||
"tinvest: %s accounts, %s operations, %s events, %s new instruments",
|
||||
@@ -139,7 +156,7 @@ class TinvestSource:
|
||||
cursor_after=json.dumps(new_cursors, sort_keys=True),
|
||||
counts=counts,
|
||||
warnings=warnings,
|
||||
changed=counts["events"] > 0 or counts["operations"] > 0,
|
||||
changed=counts["events"] > 0 or counts["operations"] > 0 or counts["sectors"] > 0,
|
||||
)
|
||||
|
||||
|
||||
@@ -288,9 +305,12 @@ async def _resolve_instruments(
|
||||
session.add(instrument)
|
||||
created += 1
|
||||
else:
|
||||
# an instrument already known from another source gains its T-Invest identity
|
||||
# an instrument already known from another source gains its T-Invest identity,
|
||||
# and the attributes only T-Invest states (MOEX gives us neither)
|
||||
instrument.tinvest_uid = instrument.tinvest_uid or info.uid
|
||||
instrument.figi = instrument.figi or info.figi
|
||||
instrument.sector = instrument.sector or info.sector
|
||||
instrument.country = instrument.country or info.country
|
||||
await session.flush()
|
||||
known[asked_uid] = instrument.id
|
||||
known[info.uid] = instrument.id
|
||||
@@ -312,6 +332,48 @@ async def _resolve_instruments(
|
||||
return {"by_uid": known, "created": created}
|
||||
|
||||
|
||||
async def _backfill_sectors(session: AsyncSession, client: TinvestClient) -> int:
|
||||
"""Fill sector (and country) on instruments that were resolved without them.
|
||||
|
||||
Uid resolution above never revisits an instrument we already know, so a paper imported
|
||||
before the per-type listing was read would otherwise keep `sector = NULL` forever. The
|
||||
listing is fetched at most once per kind per run, so this costs one RPC per kind no
|
||||
matter how many instruments are missing. A paper the listing does not carry — delisted,
|
||||
or traded outside the exchange — stays NULL rather than being guessed at; it is simply
|
||||
re-checked on the next run.
|
||||
"""
|
||||
instruments = (
|
||||
(
|
||||
await session.execute(
|
||||
select(Instrument).where(
|
||||
Instrument.tinvest_uid.is_not(None),
|
||||
Instrument.sector.is_(None),
|
||||
Instrument.asset_class.in_(list(SECTOR_KINDS)),
|
||||
)
|
||||
)
|
||||
)
|
||||
.scalars()
|
||||
.all()
|
||||
)
|
||||
filled = 0
|
||||
for instrument in instruments:
|
||||
match = await client.reference_instrument(
|
||||
SECTOR_KINDS[instrument.asset_class],
|
||||
uid=instrument.tinvest_uid,
|
||||
figi=instrument.figi,
|
||||
ticker=instrument.ticker,
|
||||
class_code=instrument.board,
|
||||
)
|
||||
if match is None or match.sector is None:
|
||||
continue
|
||||
instrument.sector = match.sector
|
||||
instrument.country = instrument.country or match.country
|
||||
filled += 1
|
||||
if instruments:
|
||||
log.info("tinvest: sector filled for %d of %d instruments", filled, len(instruments))
|
||||
return filled
|
||||
|
||||
|
||||
def _identities(operations: list[Operation]) -> dict[str, tuple[str | None, str | None]]:
|
||||
"""instrument_uid -> (figi, position_uid), as the operations themselves report it."""
|
||||
out: dict[str, tuple[str | None, str | None]] = {}
|
||||
|
||||
Reference in New Issue
Block a user