mirror of
https://github.com/ada-dmitry/osint_parser_ada.git
synced 2026-09-24 01:10:20 +00:00
feat: Add data models for disciplinary actions, organizations, and training centers
- Implemented DisciplinaryAction model for tracking disciplinary measures. - Created Organization model to represent auditing organizations with relevant details. - Developed TrainingCenter model for educational and methodological centers. - Added methods to convert models to dictionaries for easy export. feat: Implement parsers for auditors and organizations - Developed AuditorsParser to scrape and parse auditor registry data. - Created OrganizationsParser for scraping auditing organizations' registry. - Implemented GenericRegistryParser for handling generic tabular data. chore: Set up base parser functionality - Established BaseParser class with common methods for making requests and parsing HTML. - Added pagination handling and detailed page parsing capabilities. feat: Implement Excel export functionality - Created ExcelExporter class for exporting data to Excel files with formatting options. - Added support for exporting multiple sheets and organization data specifically. chore: Add logging utilities - Implemented setup_logger function for consistent logging across the application. - Configured logging to output to both console and file. chore: Update requirements and scripts - Added necessary dependencies for parsing, Excel handling, and logging. - Created run.sh and run.fish scripts for easy execution of the parser.
This commit is contained in:
@@ -0,0 +1,51 @@
|
||||
"""
|
||||
Настройка логирования
|
||||
"""
|
||||
|
||||
import logging
|
||||
import os
|
||||
from datetime import datetime
|
||||
from config import LOGGING_CONFIG
|
||||
|
||||
|
||||
def setup_logger(name: str = "sro_parser") -> logging.Logger:
|
||||
"""
|
||||
Настройка логгера для парсера
|
||||
|
||||
Args:
|
||||
name: Имя логгера
|
||||
|
||||
Returns:
|
||||
Настроенный логгер
|
||||
"""
|
||||
# Создание директории для логов
|
||||
log_dir = LOGGING_CONFIG.get("log_dir", "logs/")
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
|
||||
# Создание логгера
|
||||
logger = logging.getLogger(name)
|
||||
logger.setLevel(LOGGING_CONFIG.get("level", "INFO"))
|
||||
|
||||
# Очистка существующих обработчиков
|
||||
if logger.hasHandlers():
|
||||
logger.handlers.clear()
|
||||
|
||||
# Формат логов
|
||||
formatter = logging.Formatter(LOGGING_CONFIG.get("format"))
|
||||
|
||||
# Обработчик для консоли
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.INFO)
|
||||
console_handler.setFormatter(formatter)
|
||||
logger.addHandler(console_handler)
|
||||
|
||||
# Обработчик для файла
|
||||
log_file = os.path.join(
|
||||
log_dir, f"parser_{datetime.now().strftime('%Y-%m-%d')}.log"
|
||||
)
|
||||
file_handler = logging.FileHandler(log_file, encoding="utf-8")
|
||||
file_handler.setLevel(logging.DEBUG)
|
||||
file_handler.setFormatter(formatter)
|
||||
logger.addHandler(file_handler)
|
||||
|
||||
return logger
|
||||
Reference in New Issue
Block a user