{
  "metadata": {
    "kernelspec": {
      "display_name": "Python 3",
      "name": "python3",
      "language": "python"
    },
    "language_info": {
      "name": "python",
      "version": "3.12.13",
      "codemirror_mode": {
        "name": "ipython",
        "version": 3
      },
      "pygments_lexer": "ipython3",
      "mimetype": "text/x-python",
      "nbconvert_exporter": "python",
      "file_extension": ".py"
    },
    "colab": {
      "provenance": []
    }
  },
  "nbformat": 4,
  "nbformat_minor": 5,
  "cells": [
    {
      "cell_type": "code",
      "id": "2e2f8a4c-7063-4650-ace5-8f230c8d994a",
      "metadata": {
        "id": "5tecyK1Y5y_0"
      },
      "execution_count": 1,
      "source": [
        "import pandas as pd\n",
        "import seaborn as sns\n",
        "import matplotlib.pyplot as plt\n",
        "import numpy as np\n",
        "from scipy.stats import pearsonr, spearmanr, kendalltau\n",
        "import sqlite3\n",
        "import kagglehub\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "/home/ada/Documents/Files/files_mephi/Работы/ИРФМ/.venv/lib/python3.12/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not\n",
            " found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
            "  from .autonotebook import tqdm as notebook_tqdm\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "8f16c784-75c9-44a1-8f9c-6555bdf38a08",
      "metadata": {
        "id": "-kxc18fy5gMU",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "3db947c3-f4c3-4449-91eb-1a5722ba2deb"
      },
      "execution_count": 2,
      "source": [
        "# https://www.kaggle.com/datasets/ziya07/financial-transaction-dataset-for-risk-prediction?select=financial_transactions.csv\n",
        "dataset_dir = kagglehub.dataset_download(\n",
        "    \"mchirico/montcoalert\",\n",
        "    output_dir=\"../data/montcoalert\",\n",
        "    # force_download=True,  # раскомментируйте, если нужно перекачать датасет\n",
        ")\n",
        "\n",
        "print(\"Path to dataset files:\", dataset_dir)\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Path to dataset files: ../data/montcoalert\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "8228deb7-1ef3-4b6a-b16d-795c5b80a10b",
      "metadata": {
        "id": "jjxxafuj5twF",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 597
        },
        "outputId": "07ffb386-df17-4830-daa5-20e36a68e0c5"
      },
      "execution_count": 3,
      "source": [
        "import os\n",
        "\n",
        "df = pd.read_csv(os.path.join(dataset_dir, \"911.csv\"))\n",
        "df\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "bcd5db7c-b149-4e05-a1c5-d5df628572d4",
      "metadata": {
        "id": "JFENAQ6x6M-2",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        },
        "outputId": "69f2c9f2-256e-4184-f6cc-7895b3065557"
      },
      "execution_count": 4,
      "source": [
        "df = df.drop(columns=['desc', 'zip', 'addr', 'e'])\n",
        "df['town'] = df['twp']\n",
        "df = df.drop(columns=['twp'])\n",
        "df\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "8322a532-24e5-406f-b019-cbf264676259",
      "metadata": {
        "id": "kjx88uH06yXh",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        },
        "outputId": "b4525822-dc37-4baf-ef06-2429efb9d9b9"
      },
      "execution_count": 5,
      "source": [
        "df.isnull().sum()\n"
      ],
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "lat            0\n",
              "lng            0\n",
              "title          0\n",
              "timeStamp      0\n",
              "town         293\n",
              "dtype: int64\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "9ea98f73-7ab5-4b31-a3c7-d4031b65636c",
      "metadata": {
        "id": "CVwu4mZY7j38",
        "outputId": "10c839ff-e230-4429-8ec1-dd1160e0e52a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 241
        }
      },
      "execution_count": 6,
      "source": [
        "df = df.dropna()\n",
        "df.isnull().sum()\n"
      ],
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "lat          0\n",
              "lng          0\n",
              "title        0\n",
              "timeStamp    0\n",
              "town         0\n",
              "dtype: int64\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "1e0859b5-80a7-45d3-b681-4e3f5e97944f",
      "metadata": {
        "id": "BVziGRoT74je",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "cb96454a-a04b-4ea0-d044-981fdddcc008"
      },
      "execution_count": 7,
      "source": [
        "df.describe()\n",
        "\n",
        "# штат Пенсильвания (PA) находится примерно в следующих координатах:\n",
        "# Широта (lat): от 39.7° N до 42.5° N\n",
        "# Долгота (lng): от -80.5° W до -74.7° W\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "6931cd95-26a2-43da-98f0-90d69e3a7afe",
      "metadata": {
        "id": "YZyG4DIl8uQE",
        "outputId": "0f7ec0ed-aeed-426b-f367-640113d8057d",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "execution_count": 8,
      "source": [
        "df_clean = df[\n",
        "    (df['lat'].between(39.7, 42.5)) &\n",
        "    (df['lng'].between(-80.5, -74.7))\n",
        "]\n",
        "df_clean.describe()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "76fd2b05-51a2-420d-a7c4-6a41cf922cfb",
      "metadata": {
        "id": "9UHU8d6s8597",
        "outputId": "f822f2bb-1bcb-44b2-9371-e1bd0480a509",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 424
        }
      },
      "execution_count": 9,
      "source": [
        "df_grouped = df_clean.groupby('town').size().reset_index(name='count')\n",
        "df_grouped = df_grouped[df_grouped['count'] >= 5]\n",
        "df_grouped.sort_values(by='count', ascending=False)\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "7204f62a-c6a3-4b60-8a6c-ae055c771276",
      "metadata": {
        "id": "8XxEu-5I_6oz",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526
        },
        "outputId": "831e0639-0ffe-4a9d-e309-e6d19745375a"
      },
      "execution_count": 10,
      "source": [
        "plt.figure(figsize=(8, 6))\n",
        "sns.boxplot(y=df_grouped['count'])\n",
        "plt.title('Распределение количества вызовов по городам')\n",
        "plt.ylabel('Количество вызовов')\n",
        "plt.show()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "8e69711f-ea64-4139-bd42-bab3ffe02750",
      "metadata": {
        "id": "hL-4vAqy_gsN",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        },
        "outputId": "50f07e22-0a9c-4b43-87a4-94c18e7601e6"
      },
      "execution_count": 11,
      "source": [
        "df_grouped.describe()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "2f9d95cf-4449-4b59-918e-7662c1eba9da",
      "metadata": {
        "id": "lFjeZjICF-C-",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "9c2c08fa-ae97-4b74-f3a4-92148999c037"
      },
      "execution_count": 12,
      "source": [
        "df_grouped = df_grouped.sort_values(by='count')\n",
        "counts = df_grouped['count'].values\n",
        "towns = df_grouped['town'].values\n",
        "n = len(counts)\n",
        "\n",
        "total_range = counts.max() - counts.min() # Размах\n",
        "gap_threshold = 0.1 * total_range # Порог\n",
        "\n",
        "# Отсекаем нижние и верхние 10% по порядку наблюдений\n",
        "bottom_10_idx = int(n * 0.10)\n",
        "top_10_idx = int(n * 0.90)\n",
        "\n",
        "towns_to_drop = set()\n",
        "\n",
        "# Проверка нижних 10% (идем от центра к краям)\n",
        "# Ищем разрыв между y (индекс i) и соседом ближе к центру (индекс i+1)\n",
        "for i in range(bottom_10_idx, -1, -1): # (6, 5, 4, 3, 2, 1, 0)\n",
        "    if i + 1 < n:\n",
        "        gap = counts[i+1] - counts[i]\n",
        "        if gap > gap_threshold:\n",
        "            # Нашли разрыв. Удаляем всё от y до края\n",
        "            for j in range(i, -1, -1):\n",
        "                towns_to_drop.add(towns[j])\n",
        "            break\n",
        "\n",
        "# Проверка верхних 10% (идем от центра к краям)\n",
        "# Ищем разрыв между y (индекс i) и соседом ближе к центру (индекс i-1)\n",
        "for i in range(top_10_idx, n):\n",
        "    if i - 1 >= 0:\n",
        "        gap = counts[i] - counts[i-1]\n",
        "        if gap > gap_threshold:\n",
        "            # Нашли разрыв. Удаляем всё от y до края\n",
        "            for j in range(i, n):\n",
        "                towns_to_drop.add(towns[j])\n",
        "            break\n",
        "\n",
        "towns_to_drop\n"
      ],
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "{'LOWER MERION'}\n"
            ]
          },
          "metadata": {}
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "401fa973-2d94-440b-ae9a-65b7fa31c5e2",
      "metadata": {
        "id": "2Nch3p_tHwOf",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "cd95841f-df8d-496f-a39e-a6c4877e067a"
      },
      "execution_count": 13,
      "source": [
        "df_clean = df_clean[~df_clean['town'].isin(towns_to_drop)]\n",
        "\n",
        "df_clean.info()\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.DataFrame'>\n",
            "Index: 607557 entries, 0 to 663520\n",
            "Data columns (total 5 columns):\n",
            " #   Column     Non-Null Count   Dtype\n",
            "---  ------     --------------   -----\n",
            " 0   lat        607557 non-null  float64\n",
            " 1   lng        607557 non-null  float64\n",
            " 2   title      607557 non-null  str\n",
            " 3   timeStamp  607557 non-null  str\n",
            " 4   town       607557 non-null  str\n",
            "dtypes: float64(2), str(3)\n",
            "memory usage: 27.8 MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "a71317e7-7d15-4878-88f0-f201c15bf64d",
      "metadata": {
        "id": "pORHOk6xAe6b",
        "outputId": "4151cbca-371f-4cab-d8e1-b91e9712703a",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 526
        }
      },
      "execution_count": 14,
      "source": [
        "df_clean_grouped = df_clean.groupby('town').size().reset_index(name='count')\n",
        "plt.figure(figsize=(8, 6))\n",
        "sns.boxplot(y=df_clean_grouped['count'])\n",
        "plt.title('Распределение количества вызовов по городам')\n",
        "plt.ylabel('Количество вызовов')\n",
        "plt.show()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "29af2c22-e4b2-401d-80bc-43c812867c4e",
      "metadata": {
        "id": "xFeEAU27DVWV",
        "outputId": "d26f3a5f-6857-45e7-fc33-dce0818419f1",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 300
        }
      },
      "execution_count": 15,
      "source": [
        "df_clean_grouped.describe()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "6c425257-1b4d-41e8-8c17-1b26ade1bbeb",
      "metadata": {
        "id": "bace27bc",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "6f3cd65b-abb6-4e9d-c880-c0d3162bb064"
      },
      "execution_count": 16,
      "source": [
        "df_clean['timeStamp'] = pd.to_datetime(df_clean['timeStamp'])\n",
        "df_clean.info()\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "<class 'pandas.DataFrame'>\n",
            "Index: 607557 entries, 0 to 663520\n",
            "Data columns (total 5 columns):\n",
            " #   Column     Non-Null Count   Dtype\n",
            "---  ------     --------------   -----\n",
            " 0   lat        607557 non-null  float64\n",
            " 1   lng        607557 non-null  float64\n",
            " 2   title      607557 non-null  str\n",
            " 3   timeStamp  607557 non-null  datetime64[us]\n",
            " 4   town       607557 non-null  str\n",
            "dtypes: datetime64[us](1), float64(2), str(2)\n",
            "memory usage: 27.8 MB\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "cc0a488b-ce3f-4860-ac73-cc086623ad68",
      "metadata": {
        "id": "9d0eeaf8",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "d158ac82-85e5-464c-854b-f1264a32406b"
      },
      "execution_count": 17,
      "source": [
        "df_clean['hour'] = df_clean['timeStamp'].dt.hour\n",
        "df_clean.head()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "36b114ae-9647-4555-84ad-96cc3ca7ace4",
      "metadata": {
        "id": "91a1edc3",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "outputId": "04806088-c1be-4fae-e382-0240b286ffe5"
      },
      "execution_count": 18,
      "source": [
        "calls_by_hour = df_clean.groupby('hour').size().reset_index(name='count')\n",
        "calls_by_hour.head()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "153435e0-d311-47ce-9934-527aa4918935",
      "metadata": {
        "id": "0ef31758",
        "outputId": "4a0e5512-4174-4037-f749-697c2ca60963",
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 607
        }
      },
      "execution_count": 19,
      "source": [
        "plt.figure(figsize=(10, 6))\n",
        "sns.lineplot(x='hour', y='count', data=calls_by_hour, marker='o', label='Calls Count')\n",
        "plt.title('Общее количество вызовов 911 по часам суток')\n",
        "plt.xlabel('Час суток')\n",
        "plt.ylabel('Количество вызовов')\n",
        "plt.xticks(calls_by_hour['hour'])\n",
        "plt.grid(True, linestyle='--', alpha=0.7)\n",
        "plt.legend()\n",
        "plt.tight_layout()\n",
        "plt.show()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "fe76bee3-d0a0-4dad-9867-ce6c3c711d08",
      "metadata": {
        "id": "jg4PAmmUQ_9o",
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "512127d4-5308-419f-a7dd-cab7399473af"
      },
      "execution_count": 20,
      "source": [
        "def analyze_correlations(df, col1, col2):\n",
        "    x = df[col1].values\n",
        "    y = df[col2].values\n",
        "\n",
        "    results = {\n",
        "        'Пирсон': pearsonr(x, y),\n",
        "        'Спирмен': spearmanr(x, y),\n",
        "        'Кендалл': kendalltau(x, y)\n",
        "    }\n",
        "\n",
        "    for method, (corr, p_value) in results.items():\n",
        "        abs_corr = abs(corr)\n",
        "\n",
        "        if method == 'Кендалл':\n",
        "            thresholds = [0.2, 0.4, 0.6, 0.8]\n",
        "        else:\n",
        "            thresholds = [0.3, 0.5, 0.7, 0.9]\n",
        "\n",
        "        if abs_corr < thresholds[0]:\n",
        "            strength = \"слабая (или отсутствует)\"\n",
        "        elif abs_corr < thresholds[1]:\n",
        "            strength = \"умеренная\"\n",
        "        elif abs_corr < thresholds[2]:\n",
        "            strength = \"заметная\"\n",
        "        elif abs_corr < thresholds[3]:\n",
        "            strength = \"высокая\"\n",
        "        else:\n",
        "            strength = \"весьма высокая\"\n",
        "\n",
        "        status = \"Значима\" if p_value < 0.05 else \"НЕ значима\"\n",
        "        direction = \"положительная\" if corr > 0 else \"отрицательная\"\n",
        "\n",
        "        print(f\"[{method}]\")\n",
        "        print(f\"  Коэффициент: {corr:.4f} ({direction})\")\n",
        "        print(f\"  P-value:     {p_value:.4e} ({status})\")\n",
        "\n",
        "        if p_value < 0.05:\n",
        "            print(f\"  Вывод: Наблюдается {strength} связь.\")\n",
        "        else:\n",
        "            print(f\"  Вывод: Недостаточно данных для подтверждения связи.\")\n",
        "        print(\"-\" * 40)\n",
        "\n",
        "analyze_correlations(calls_by_hour, 'hour', 'count')\n"
      ],
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "[Пирсон]\n",
            "  Коэффициент: 0.5114 (положительная)\n",
            "  P-value:     1.0638e-02 (Значима)\n",
            "  Вывод: Наблюдается заметная связь.\n",
            "----------------------------------------\n",
            "[Спирмен]\n",
            "  Коэффициент: 0.4991 (положительная)\n",
            "  P-value:     1.3028e-02 (Значима)\n",
            "  Вывод: Наблюдается умеренная связь.\n",
            "----------------------------------------\n",
            "[Кендалл]\n",
            "  Коэффициент: 0.3623 (положительная)\n",
            "  P-value:     1.2905e-02 (Значима)\n",
            "  Вывод: Наблюдается умеренная связь.\n",
            "----------------------------------------\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "id": "8ee4c692-afde-4bad-b7a5-bb0dda83a5aa",
      "metadata": {},
      "execution_count": 24,
      "source": [
        "conn = sqlite3.connect('../data/calls.db')\n",
        "df_clean.to_sql('calls', conn, if_exists='replace', index=False)\n",
        "conn.close()\n"
      ],
      "outputs": []
    },
    {
      "cell_type": "code",
      "id": "4e3cb3c7-62d9-4d2b-b7d0-5d209464fac2",
      "metadata": {
        "id": "1e5d291c",
        "outputId": "783c15e5-5dce-46e2-c666-1f917a832051",
        "colab": {
          "base_uri": "https://localhost:8080/"
        }
      },
      "execution_count": 25,
      "source": [
        "conn = sqlite3.connect('../data/calls.db')\n",
        "query = \"SELECT * FROM calls LIMIT 5\"\n",
        "curr = conn.cursor()\n",
        "\n",
        "data = curr.execute(query).fetchall()\n",
        "\n",
        "conn.close()\n",
        "data\n"
      ],
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "[(40.2978759,\n",
              "  -75.5812935,\n",
              "  'EMS: BACK PAINS/INJURY',\n",
              "  '2015-12-10 17:10:52',\n",
              "  'NEW HANOVER',\n",
              "  17),\n",
              " (40.2580614,\n",
              "  -75.2646799,\n",
              "  'EMS: DIABETIC EMERGENCY',\n",
              "  '2015-12-10 17:29:21',\n",
              "  'HATFIELD TOWNSHIP',\n",
              "  17),\n",
              " (40.1211818,\n",
              "  -75.3519752,\n",
              "  'Fire: GAS-ODOR/LEAK',\n",
              "  '2015-12-10 14:39:21',\n",
              "  'NORRISTOWN',\n",
              "  14),\n",
              " (40.116153,\n",
              "  -75.343513,\n",
              "  'EMS: CARDIAC EMERGENCY',\n",
              "  '2015-12-10 16:47:36',\n",
              "  'NORRISTOWN',\n",
              "  16),\n",
              " (40.251492,\n",
              "  -75.6033497,\n",
              "  'EMS: DIZZINESS',\n",
              "  '2015-12-10 16:56:52',\n",
              "  'LOWER POTTSGROVE',\n",
              "  16)]\n"
            ]
          },
          "metadata": {}
        }
      ]
    }
  ]
}