mirror of
https://github.com/ada-dmitry/osint_parser_ada.git
synced 2026-09-24 01:10:20 +00:00
ada
12d5c6eaff
- Implemented DisciplinaryAction model for tracking disciplinary measures. - Created Organization model to represent auditing organizations with relevant details. - Developed TrainingCenter model for educational and methodological centers. - Added methods to convert models to dictionaries for easy export. feat: Implement parsers for auditors and organizations - Developed AuditorsParser to scrape and parse auditor registry data. - Created OrganizationsParser for scraping auditing organizations' registry. - Implemented GenericRegistryParser for handling generic tabular data. chore: Set up base parser functionality - Established BaseParser class with common methods for making requests and parsing HTML. - Added pagination handling and detailed page parsing capabilities. feat: Implement Excel export functionality - Created ExcelExporter class for exporting data to Excel files with formatting options. - Added support for exporting multiple sheets and organization data specifically. chore: Add logging utilities - Implemented setup_logger function for consistent logging across the application. - Configured logging to output to both console and file. chore: Update requirements and scripts - Added necessary dependencies for parsing, Excel handling, and logging. - Created run.sh and run.fish scripts for easy execution of the parser.
52 lines
1.4 KiB
Python
52 lines
1.4 KiB
Python
"""
|
|
Настройка логирования
|
|
"""
|
|
|
|
import logging
|
|
import os
|
|
from datetime import datetime
|
|
from config import LOGGING_CONFIG
|
|
|
|
|
|
def setup_logger(name: str = "sro_parser") -> logging.Logger:
|
|
"""
|
|
Настройка логгера для парсера
|
|
|
|
Args:
|
|
name: Имя логгера
|
|
|
|
Returns:
|
|
Настроенный логгер
|
|
"""
|
|
# Создание директории для логов
|
|
log_dir = LOGGING_CONFIG.get("log_dir", "logs/")
|
|
os.makedirs(log_dir, exist_ok=True)
|
|
|
|
# Создание логгера
|
|
logger = logging.getLogger(name)
|
|
logger.setLevel(LOGGING_CONFIG.get("level", "INFO"))
|
|
|
|
# Очистка существующих обработчиков
|
|
if logger.hasHandlers():
|
|
logger.handlers.clear()
|
|
|
|
# Формат логов
|
|
formatter = logging.Formatter(LOGGING_CONFIG.get("format"))
|
|
|
|
# Обработчик для консоли
|
|
console_handler = logging.StreamHandler()
|
|
console_handler.setLevel(logging.INFO)
|
|
console_handler.setFormatter(formatter)
|
|
logger.addHandler(console_handler)
|
|
|
|
# Обработчик для файла
|
|
log_file = os.path.join(
|
|
log_dir, f"parser_{datetime.now().strftime('%Y-%m-%d')}.log"
|
|
)
|
|
file_handler = logging.FileHandler(log_file, encoding="utf-8")
|
|
file_handler.setLevel(logging.DEBUG)
|
|
file_handler.setFormatter(formatter)
|
|
logger.addHandler(file_handler)
|
|
|
|
return logger
|