Add loader and processor modules with documentation and output files

- Implemented loader in `modules/loader.py` to download HTML pages using curl and urllib.
- Created processor in `modules/processor.py` for processing the text of the Russian Criminal Code (УК РФ).
- Added README files for both loader and processor explaining their functionality and usage.
- Generated output files including original text, prepared text, and subject index in CSV and JSON formats.

modules/crawler.py

@@ -1,64 +1,215 @@
-from urllib.parse import urljoin, urldefrag
+import re
+import time
+from urllib.parse import urljoin, urldefrag, urlparse
+
 from lxml import html
 
 from modules.loader import download_html
 
+
+START_URL = "https://www.consultant.ru/document/cons_doc_LAW_10699/"
+FIRST_ARTICLE_URL = START_URL + "e8ecf933c52a85d9223094e0e7fbf52f0128d399/"
+
+ARTICLE_RE = re.compile(r"^\s*(?:УК РФ,?\s+)?Статья\s+\d+(?:\.\d+)?\.", re.IGNORECASE)
+STRUCTURE_PREFIX_RE = re.compile(r"^\s*(?:\d+(?:\.\d+)*|[а-яё])[\.)]\s+", re.IGNORECASE)
+SPACE_RE = re.compile(r"[ \t\r\f\v]+")
+
+
 def normalize_url(base_url: str, href: str) -> str:
-    full = urljoin(base_url, href)
-    full, _ = urldefrag(full)
-    return full
-
-def extract_links(page_html: str, base_url: str) -> list[str]:
-    """
-    Извлекает все уникальные ссылки из HTML-страницы.
-    :param page_html: HTML-код страницы
-    :param base_url: Базовый URL для нормализации ссылок
-    :return: Список уникальных нормализованных ссылок
-    """
-    tree = html.fromstring(page_html)
-    links = []
-
-    for a in tree.xpath("//a[@href]"):
-        href = a.attrib["href"].strip()
-        full_url = normalize_url(base_url, href)
-        links.append(full_url)
-
-    return list(dict.fromkeys(links))
+    url = urljoin(base_url, href)
+    url, _ = urldefrag(url)
+    return url
 
 
-def is_terminal_page(page_html: str, url: str) -> bool:
-    tree = html.fromstring(page_html)
-    links = tree.xpath("//a[contains(text(), 'УК РФ Статья')]")
-    return len(links) == 0
+def document_prefix(url: str) -> str:
+    parts = [part for part in urlparse(url).path.split("/") if part]
+    return f"/{parts[0]}/{parts[1]}/"
 
-def extract_page_text(page_html: str, url: str) -> str:
-    tree = html.fromstring(page_html)
-    text = tree.text_content()
+
+def is_same_document(url: str, prefix: str) -> bool:
+    parsed = urlparse(url)
+    return parsed.netloc in {"", "consultant.ru", "www.consultant.ru"} and parsed.path.startswith(prefix)
+
+
+def download_document_page(url: str, allow_partial: bool = False) -> str:
+    last_html = ""
+
+    for timeout in (8, 30, 30):
+        try:
+            page_html = download_html(url, retries=1, timeout=timeout)
+        except RuntimeError:
+            time.sleep(0.5)
+            continue
+
+        if allow_partial or "</html>" in page_html.lower():
+            return page_html
+
+        last_html = page_html
+        time.sleep(0.5)
+
+    if last_html:
+        return last_html
+
+    raise RuntimeError(f"Failed to download page: {url}")
+
+
+def clean_text(text: str) -> str:
+    text = text.replace("\xa0", " ")
+    text = SPACE_RE.sub(" ", text)
+    text = STRUCTURE_PREFIX_RE.sub("", text)
     return text.strip()
 
-def crawl_document(start_url, is_terminal_page=is_terminal_page, extract_document_links=extract_links, extract_page_text=extract_page_text):
-    visited = set()
-    pages = []
 
-    def dfs(url):
-        if url in visited:
-            return
-        visited.add(url)
+def main_content(tree):
+    nodes = tree.xpath("//div[contains(concat(' ', normalize-space(@class), ' '), ' document-page__content ')]")
+    return nodes[0] if nodes else None
+
+
+def article_title(page_html: str) -> str:
+    tree = html.fromstring(page_html)
+    content = main_content(tree)
+    if content is None:
+        return ""
+
+    h1 = content.xpath(".//h1")
+    if h1 and clean_text(h1[0].text_content()):
+        return clean_text(h1[0].text_content())
+
+    styles = content.xpath(".//div[contains(concat(' ', normalize-space(@class), ' '), ' doc-style ')]")
+    return clean_text(styles[0].text_content()) if styles else ""
+
+
+def is_article_page(page_html: str) -> bool:
+    return bool(ARTICLE_RE.match(article_title(page_html)))
+
+
+def service_line(line: str) -> bool:
+    low = line.lower()
+
+    if low.startswith("(см. текст") or low.startswith("(в ред.") or low.startswith("(введен"):
+        return True
+    if low.startswith("(част") and ("в ред." in low or "введен" in low):
+        return True
+    if "федеральн" in low and any(word in low for word in ("ред.", "введен", "утратил", "утратила")):
+        return True
+    if low in {"президент", "российской федерации", "б.ельцин", "москва, кремль"}:
+        return True
+    if re.fullmatch(r"\d{1,2}\s+[а-яё]+\s+\d{4}\s+года", low):
+        return True
+    if re.fullmatch(r"n\s+\d+\s*-\s*фз", low):
+        return True
+
+    return False
+
+
+def extract_page_text(page_html: str, url: str = "") -> str:
+    tree = html.fromstring(page_html)
+    content = main_content(tree)
+    if content is None:
+        return ""
+
+    for node in content.xpath(
+        ".//h1"
+        " | .//div[contains(concat(' ', normalize-space(@class), ' '), ' doc-style ')]"
+        " | .//div[contains(concat(' ', normalize-space(@class), ' '), ' doc-insert ')]"
+        " | .//div[contains(concat(' ', normalize-space(@class), ' '), ' doc-roll ')]"
+    ):
+        node.getparent().remove(node)
+
+    lines = []
+    for p in content.xpath(".//p"):
+        line = clean_text(p.text_content())
+        if line and not service_line(line):
+            lines.append(line)
+
+    return "\n".join(lines)
+
+
+def extract_article_urls(page_html: str, base_url: str) -> list[str]:
+    tree = html.fromstring(page_html)
+    prefix = document_prefix(base_url)
+    result = []
+
+    for link in tree.xpath("//a[@href]"):
+        text = clean_text(link.text_content())
+        url = normalize_url(base_url, link.attrib["href"])
+        if ARTICLE_RE.match(text) and is_same_document(url, prefix) and url not in result:
+            result.append(url)
+
+    return result
+
+
+def extract_next_document_url(page_html: str, base_url: str) -> str | None:
+    tree = html.fromstring(page_html)
+    prefix = document_prefix(base_url)
+    links = tree.xpath("//a[contains(concat(' ', normalize-space(@class), ' '), ' pages__right ')][@href]")
+    if not links:
+        return None
+
+    url = normalize_url(base_url, links[0].attrib["href"])
+    return url if is_same_document(url, prefix) else None
+
+
+def next_known_article(current_url: str, known_urls: list[str], visited: set[str]) -> str | None:
+    if current_url not in known_urls:
+        return None
+
+    current_index = known_urls.index(current_url)
+    for url in known_urls[current_index + 1:]:
+        if url not in visited:
+            return url
+
+    return None
+
+
+def add_known_urls(known_urls: list[str], urls: list[str]) -> None:
+    for url in urls:
+        if url not in known_urls:
+            known_urls.append(url)
+
+
+def crawl_document(start_url: str = START_URL, max_pages: int = 800) -> list[dict]:
+    pages = []
+    visited = set()
+    known_urls = []
+
+    try:
+        start_html = download_document_page(start_url, allow_partial=True)
+        add_known_urls(known_urls, extract_article_urls(start_html, start_url))
+        current_url = known_urls[0] if known_urls else FIRST_ARTICLE_URL
+    except RuntimeError as exc:
+        print(f"[crawler] start page is unavailable: {exc}")
+        current_url = FIRST_ARTICLE_URL
+        add_known_urls(known_urls, [FIRST_ARTICLE_URL])
+
+    while current_url and current_url not in visited and len(visited) < max_pages:
+        visited.add(current_url)
 
         try:
-            page_html = download_html(url)
+            page_html = download_document_page(current_url)
         except RuntimeError as exc:
-            print(f"[crawler] skip unreachable page: {url} ({exc})")
-            return
+            print(f"[crawler] skip page {current_url}: {exc}")
+            break
 
-        if is_terminal_page(page_html, url):
-            text = extract_page_text(page_html, url)
-            pages.append({"url": url, "text": text})
-            return
+        add_known_urls(known_urls, extract_article_urls(page_html, current_url))
 
-        child_links = extract_document_links(page_html, url)
-        for link in child_links:
-            dfs(link)
+        if is_article_page(page_html):
+            text = extract_page_text(page_html, current_url)
+            if text:
+                pages.append(
+                    {
+                        "url": current_url,
+                        "title": article_title(page_html),
+                        "text": text,
+                    }
+                )
 
-    dfs(start_url)
-    return pages
+        if "Статья 361." in article_title(page_html):
+            break
+
+        next_url = extract_next_document_url(page_html, current_url)
+        if next_url is None:
+            next_url = next_known_article(current_url, known_urls, visited)
+        current_url = next_url
+
+    return pages

modules/loader.py

@@ -1,63 +1,62 @@
-from http.client import IncompleteRead
-import socket
-from urllib.error import URLError
-from urllib.request import Request, urlopen
 import ssl
+import subprocess
 import time
+from urllib.parse import urlparse
+from urllib.request import Request, urlopen
 
 
-def _read_response_bytes(response, chunk_size: int = 64 * 1024, min_partial_bytes: int = 4096) -> bytes:
-    """Читает ответ по частям; при таймауте возвращает уже полученные байты, если их достаточно."""
-    chunks = []
-    total = 0
+HEADERS = {
+    "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 Chrome/124.0 Safari/537.36",
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "ru,en-US;q=0.9,en;q=0.8",
+}
 
-    while True:
-        try:
-            chunk = response.read(chunk_size)
-        except (TimeoutError, socket.timeout, OSError):
-            if total >= min_partial_bytes:
-                return b"".join(chunks)
-            raise
 
-        if not chunk:
-            break
-
-        chunks.append(chunk)
-        total += len(chunk)
-
-    return b"".join(chunks)
-
-def download_html(url: str, retries: int = 5, timeout: float = 40.0) -> str:
-    """Загружает HTML с повторными попытками при временных сетевых ошибках."""
-    ctx = ssl.create_default_context()
-    ctx.check_hostname = False
-    ctx.verify_mode = ssl.CERT_NONE
-
-    req = Request(
+def download_with_curl(url: str, timeout: float) -> str:
+    command = [
+        "curl",
+        "-L",
+        "--compressed",
+        "--silent",
+        "--show-error",
+        "--max-time",
+        str(int(timeout)),
+        "-A",
+        HEADERS["User-Agent"],
         url,
-        headers={
-            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
-            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
-            "Accept-Encoding": "identity",
-            "Accept-Language": "ru,en-US;q=0.9,en;q=0.8",
-            "Connection": "close",
-        },
-    )
+    ]
+    result = subprocess.run(command, capture_output=True, check=False)
+
+    if result.stdout:
+        return result.stdout.decode("utf-8", errors="ignore")
+
+    message = result.stderr.decode("utf-8", errors="ignore").strip()
+    raise RuntimeError(message or f"curl finished with code {result.returncode}")
+
+
+def download_with_urllib(url: str, timeout: float) -> str:
+    context = ssl.create_default_context()
+    context.check_hostname = False
+    context.verify_mode = ssl.CERT_NONE
+
+    request = Request(url, headers=HEADERS)
+    with urlopen(request, context=context, timeout=timeout) as response:
+        return response.read().decode("utf-8", errors="ignore")
+
+
+def download_html(url: str, retries: int = 3, timeout: float = 30.0) -> str:
+    """Загружает HTML. Для ConsultantPlus сначала пробует curl, для остальных сайтов urllib."""
+    is_consultant = urlparse(url).netloc.endswith("consultant.ru")
+    loaders = [download_with_curl, download_with_urllib] if is_consultant else [download_with_urllib, download_with_curl]
     last_error = None
 
-    for attempt in range(retries):
-        try:
-            with urlopen(req, context=ctx, timeout=timeout) as response:
-                raw = _read_response_bytes(response)
-                return raw.decode("utf-8", errors="ignore")
-        except IncompleteRead as exc:
-            if exc.partial:
-                return exc.partial.decode("utf-8", errors="ignore")
-            last_error = exc
-        except (TimeoutError, socket.timeout, URLError, OSError) as exc:
-            last_error = exc
-            if attempt == retries - 1:
-                break
-            time.sleep(0.5 * (2 ** attempt))
+    for _ in range(retries):
+        for loader in loaders:
+            try:
+                return loader(url, timeout)
+            except Exception as exc:
+                last_error = exc
 
-    raise RuntimeError(f"Failed to download URL after {retries} attempts: {url}") from last_error
+        time.sleep(0.5)
+
+    raise RuntimeError(f"Failed to download URL: {url}") from last_error

modules/processor.py

@@ -0,0 +1,239 @@
+from collections import Counter
+from functools import lru_cache
+import csv
+import json
+from pathlib import Path
+import re
+
+
+WORD_RE = re.compile(r"[А-Яа-яЁё]+(?:-[А-Яа-яЁё]+)?")
+DIGIT_BEFORE_WORD_RE = re.compile(r"\d[\d\s.,-]*$")
+
+STOP_WORDS = set(
+    """
+    а без более бы бывший был была были было быть в во весь вместе вне вновь все всего
+    всей всем всеми всех вследствие вы где да для до его ее если есть же за из или им
+    ими их к как ко когда который ли либо между менее мочь на над надо наиболее
+    настоящий не него нее нет ни них но о об оба однако он она они оно от перед по под
+    после при про с со свой себя так также такой там то того той только том тот у уже
+    чем через что чтобы эта эти это этот являться абзац глава данный кодекс пункт
+    раздел статья часть
+    """.split()
+)
+
+NUMERAL_WORDS = set(
+    """
+    ноль один два три четыре пять шесть семь восемь девять десять одиннадцать
+    двенадцать тринадцать четырнадцать пятнадцать шестнадцать семнадцать
+    восемнадцать девятнадцать двадцать тридцать сорок пятьдесят шестьдесят
+    семьдесят восемьдесят девяносто сто двести триста четыреста пятьсот шестьсот
+    семьсот восемьсот девятьсот тысяча миллион миллиард триллион полтора
+    полтораста оба
+    """.split()
+)
+
+NUMBER_UNITS = set(
+    """
+    год месяц неделя день сутки час минута рубль копейка процент метр километр грамм
+    килограмм литр
+    """.split()
+)
+
+PROPER_WORDS = set(
+    """
+    ельцин интернет конституция кремль москва россия рф снг ссср
+    """.split()
+)
+
+PROPER_PHRASES = [
+    phrase.split()
+    for phrase in [
+        "арбитражный процессуальный кодекс",
+        "верховный суд",
+        "вооруженный сила",
+        "государственный дума",
+        "гражданский кодекс",
+        "евразийский экономический союз",
+        "земельный кодекс",
+        "конституционный суд",
+        "конституция российский федерация",
+        "международный уголовный суд",
+        "налоговый кодекс",
+        "организация объединить нация",
+        "правительство российский федерация",
+        "президент российский федерация",
+        "российский федерация",
+        "совет безопасность",
+        "совет федерация",
+        "содружество независимый государство",
+        "таможенный союз",
+        "трудовой кодекс",
+        "уголовно-исполнительный кодекс",
+        "уголовно-процессуальный кодекс",
+        "уголовный кодекс",
+        "федеральный закон",
+        "федеральный собрание",
+        "центральный банк",
+    ]
+]
+
+try:
+    import pymorphy3
+except ImportError:
+    MORPH = None
+else:
+    MORPH = pymorphy3.MorphAnalyzer()
+
+
+def normalize(word: str) -> str:
+    return word.lower().replace("ё", "е")
+
+
+@lru_cache(maxsize=100_000)
+def parse_word(word: str):
+    if MORPH is None:
+        return normalize(word), ""
+
+    parsed = MORPH.parse(normalize(word))
+    if not parsed:
+        return normalize(word), ""
+
+    best = parsed[0]
+    return normalize(best.normal_form), str(best.tag)
+
+
+def has_tag(tag: str, names: set[str]) -> bool:
+    return bool(set(re.split(r"[, ]+", tag)) & names)
+
+
+def is_numeral(lemma: str, tag: str) -> bool:
+    return lemma in NUMERAL_WORDS or has_tag(tag, {"NUMR", "Anum"})
+
+
+def is_proper_name(lemma: str, tag: str) -> bool:
+    return lemma in PROPER_WORDS or has_tag(tag, {"Name", "Surn", "Patr", "Geox", "Orgn"})
+
+
+def find_phrase_positions(lemmas: list[str]) -> set[int]:
+    positions = set()
+
+    for phrase in PROPER_PHRASES:
+        phrase_len = len(phrase)
+        for start in range(len(lemmas) - phrase_len + 1):
+            if lemmas[start : start + phrase_len] == phrase:
+                positions.update(range(start, start + phrase_len))
+
+    return positions
+
+
+def words_from_line(line: str):
+    words = []
+
+    for match in WORD_RE.finditer(line):
+        word = match.group()
+        lemma, tag = parse_word(word)
+        before_word = line[: match.start()].rstrip()
+        words.append(
+            {
+                "source": word,
+                "lemma": lemma,
+                "tag": tag,
+                "char": match.start() + 1,
+                "after_digit": bool(DIGIT_BEFORE_WORD_RE.search(before_word)),
+            }
+        )
+
+    return words
+
+
+def is_bad_word(word: dict, phrase_positions: set[int], index: int, after_number: bool) -> bool:
+    lemma = word["lemma"]
+    tag = word["tag"]
+
+    if len(lemma) <= 2 or lemma in STOP_WORDS:
+        return True
+    if index in phrase_positions or is_proper_name(lemma, tag):
+        return True
+    if is_numeral(lemma, tag):
+        return True
+    if lemma in NUMBER_UNITS and (after_number or word["after_digit"]):
+        return True
+
+    return False
+
+
+def good_words_from_line(line: str):
+    words = words_from_line(line)
+    phrase_positions = find_phrase_positions([word["lemma"] for word in words])
+    after_number = False
+
+    for index, word in enumerate(words):
+        bad = is_bad_word(word, phrase_positions, index, after_number)
+        after_number = is_numeral(word["lemma"], word["tag"]) or (
+            word["lemma"] in NUMBER_UNITS and (after_number or word["after_digit"])
+        )
+
+        if not bad:
+            yield word
+
+
+def prepared_terms(text: str):
+    for line_number, line in enumerate(text.splitlines(), start=1):
+        for word in good_words_from_line(line):
+            yield word["lemma"], line_number, word["char"], word["source"]
+
+
+def prepare_text(text: str) -> str:
+    lines = []
+
+    for line in text.splitlines():
+        lemmas = [word["lemma"] for word in good_words_from_line(line)]
+        if lemmas:
+            lines.append(" ".join(lemmas))
+
+    return "\n".join(lines)
+
+
+def build_subject_index(text: str, top_n: int = 100) -> list[dict]:
+    counts = Counter()
+    first_place = {}
+
+    for lemma, line, char, source in prepared_terms(text):
+        counts[lemma] += 1
+        first_place.setdefault(lemma, (line, char, source))
+
+    result = []
+    for lemma, count in counts.most_common(top_n):
+        line, char, source = first_place[lemma]
+        result.append(
+            {
+                "word": lemma,
+                "count": count,
+                "line": line,
+                "char": char,
+                "source_word": source,
+            }
+        )
+
+    return result
+
+
+def write_subject_index_csv(entries: list[dict], path: Path) -> None:
+    with path.open("w", encoding="utf-8", newline="") as file:
+        writer = csv.writer(file, delimiter=";")
+        writer.writerow(["word", "count", "line", "char", "source_word"])
+        for entry in entries:
+            writer.writerow(
+                [
+                    entry["word"],
+                    entry["count"],
+                    entry["line"],
+                    entry["char"],
+                    entry["source_word"],
+                ]
+            )
+
+
+def write_subject_index_json(entries: list[dict], path: Path) -> None:
+    with path.open("w", encoding="utf-8") as file:
+        json.dump(entries, file, ensure_ascii=False, indent=2)