Add web crawler for legal documents

- Implemented a new crawler module to extract text and links from legal documents.
- Added a loader module to handle HTML downloads with error handling and retries.
- Created a main script to initiate the crawling process on a specified URL.
- Defined functions for normalizing URLs, extracting links, checking terminal pages, and extracting text content.
- Updated project metadata with dependencies and versioning in pyproject.toml and uv.lock.

modules/crawler.py

@@ -0,0 +1,64 @@
+from urllib.parse import urljoin, urldefrag
+from lxml import html
+
+from modules.loader import download_html
+
+def normalize_url(base_url: str, href: str) -> str:
+    full = urljoin(base_url, href)
+    full, _ = urldefrag(full)
+    return full
+
+def extract_links(page_html: str, base_url: str) -> list[str]:
+    """
+    Извлекает все уникальные ссылки из HTML-страницы.
+    :param page_html: HTML-код страницы
+    :param base_url: Базовый URL для нормализации ссылок
+    :return: Список уникальных нормализованных ссылок
+    """
+    tree = html.fromstring(page_html)
+    links = []
+
+    for a in tree.xpath("//a[@href]"):
+        href = a.attrib["href"].strip()
+        full_url = normalize_url(base_url, href)
+        links.append(full_url)
+
+    return list(dict.fromkeys(links))
+
+
+def is_terminal_page(page_html: str, url: str) -> bool:
+    tree = html.fromstring(page_html)
+    links = tree.xpath("//a[contains(text(), 'УК РФ Статья')]")
+    return len(links) == 0
+
+def extract_page_text(page_html: str, url: str) -> str:
+    tree = html.fromstring(page_html)
+    text = tree.text_content()
+    return text.strip()
+
+def crawl_document(start_url, is_terminal_page=is_terminal_page, extract_document_links=extract_links, extract_page_text=extract_page_text):
+    visited = set()
+    pages = []
+
+    def dfs(url):
+        if url in visited:
+            return
+        visited.add(url)
+
+        try:
+            page_html = download_html(url)
+        except RuntimeError as exc:
+            print(f"[crawler] skip unreachable page: {url} ({exc})")
+            return
+
+        if is_terminal_page(page_html, url):
+            text = extract_page_text(page_html, url)
+            pages.append({"url": url, "text": text})
+            return
+
+        child_links = extract_document_links(page_html, url)
+        for link in child_links:
+            dfs(link)
+
+    dfs(start_url)
+    return pages

modules/loader.py

@@ -0,0 +1,63 @@
+from http.client import IncompleteRead
+import socket
+from urllib.error import URLError
+from urllib.request import Request, urlopen
+import ssl
+import time
+
+
+def _read_response_bytes(response, chunk_size: int = 64 * 1024, min_partial_bytes: int = 4096) -> bytes:
+    """Читает ответ по частям; при таймауте возвращает уже полученные байты, если их достаточно."""
+    chunks = []
+    total = 0
+
+    while True:
+        try:
+            chunk = response.read(chunk_size)
+        except (TimeoutError, socket.timeout, OSError):
+            if total >= min_partial_bytes:
+                return b"".join(chunks)
+            raise
+
+        if not chunk:
+            break
+
+        chunks.append(chunk)
+        total += len(chunk)
+
+    return b"".join(chunks)
+
+def download_html(url: str, retries: int = 5, timeout: float = 40.0) -> str:
+    """Загружает HTML с повторными попытками при временных сетевых ошибках."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+
+    req = Request(
+        url,
+        headers={
+            "User-Agent": "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36",
+            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+            "Accept-Encoding": "identity",
+            "Accept-Language": "ru,en-US;q=0.9,en;q=0.8",
+            "Connection": "close",
+        },
+    )
+    last_error = None
+
+    for attempt in range(retries):
+        try:
+            with urlopen(req, context=ctx, timeout=timeout) as response:
+                raw = _read_response_bytes(response)
+                return raw.decode("utf-8", errors="ignore")
+        except IncompleteRead as exc:
+            if exc.partial:
+                return exc.partial.decode("utf-8", errors="ignore")
+            last_error = exc
+        except (TimeoutError, socket.timeout, URLError, OSError) as exc:
+            last_error = exc
+            if attempt == retries - 1:
+                break
+            time.sleep(0.5 * (2 ** attempt))
+
+    raise RuntimeError(f"Failed to download URL after {retries} attempts: {url}") from last_error