mirror of
https://github.com/ada-dmitry/CourseWork_IRFM.git
synced 2026-09-24 01:10:18 +00:00
Dmitry
0b41577a42
- Implemented loader in `modules/loader.py` to download HTML pages using curl and urllib. - Created processor in `modules/processor.py` for processing the text of the Russian Criminal Code (УК РФ). - Added README files for both loader and processor explaining their functionality and usage. - Generated output files including original text, prepared text, and subject index in CSV and JSON formats.
39 lines
1.4 KiB
Python
39 lines
1.4 KiB
Python
from modules.crawler import crawl_document
|
|
from modules.processor import (
|
|
build_subject_index,
|
|
prepare_text,
|
|
write_subject_index_csv,
|
|
write_subject_index_json,
|
|
)
|
|
|
|
|
|
SOURCE_URL = "https://www.consultant.ru/document/cons_doc_LAW_10699/"
|
|
OUTPUT_DIR = "output"
|
|
|
|
def main():
|
|
from pathlib import Path
|
|
|
|
output_dir = Path(OUTPUT_DIR)
|
|
output_dir.mkdir(exist_ok=True)
|
|
|
|
pages = crawl_document(SOURCE_URL)
|
|
original_text = "\n".join(page["text"] for page in pages)
|
|
|
|
prepared_text = prepare_text(original_text)
|
|
subject_index = build_subject_index(original_text, top_n=100)
|
|
|
|
(output_dir / "uk_rf_original.txt").write_text(original_text, encoding="utf-8")
|
|
(output_dir / "uk_rf_prepared.txt").write_text(prepared_text, encoding="utf-8")
|
|
write_subject_index_csv(subject_index, output_dir / "uk_rf_subject_index.csv")
|
|
write_subject_index_json(subject_index, output_dir / "uk_rf_subject_index.json")
|
|
|
|
print(f"Загружено статей: {len(pages)}")
|
|
print(f"Строк исходного текста: {len(original_text.splitlines())}")
|
|
print(f"Лемм после обработки: {len(prepared_text.split())}")
|
|
print("Топ-10 предметного указателя:")
|
|
for entry in subject_index[:10]:
|
|
print(f"{entry['word']}: {entry['count']} (строка {entry['line']}, символ {entry['char']})")
|
|
|
|
if __name__ == "__main__":
|
|
main()
|