{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "1b85622b",
   "metadata": {},
   "source": [
    "# Colorado Spills \n",
    "## OLS, NBR, Interrupted Time Series Analysis\n",
    "## Just top 3 Counties\n",
    "### Author: [David P. Adams, PhD](https://dadams.io)\n",
    "#### Date: 2025-07-02"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "d456a510",
   "metadata": {},
   "source": [
    "---\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9814ecd7",
   "metadata": {},
   "source": [
    "# Setup"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "eba79ece",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dadams/miniconda3/envs/spatial_env2/lib/python3.10/site-packages/pandas/io/sql.py:1725: SAWarning: Did not recognize type 'geometry' of column 'geometry'\n",
      "  self.meta.reflect(bind=self.con, only=[table_name], views=True)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "import geopandas as gpd\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "import os\n",
    "\n",
    "# Database connection details from zshrc environment variables\n",
    "#  use .env file for local development\n",
    "\n",
    "\n",
    "db_name = 'colorado_spills'\n",
    "user = os.getenv('DB_USER')\n",
    "password = os.getenv('DB_PASSWORD')\n",
    "host = os.getenv('DB_HOST')\n",
    "port = os.getenv('DB_PORT')\n",
    "\n",
    "\n",
    "# Create an engine to connect to the PostgreSQL database\n",
    "engine = create_engine(f'postgresql+psycopg2://{user}:{password}@{host}:{port}/{db_name}')\n",
    "\n",
    "# Read in the spills_with_demographics_geog as spills\n",
    "spills = pd.read_sql_table('spills_with_demographics_geog', engine)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "da21b89e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# use longitude and latitude to create a GeoDataFrame from spills data\n",
    "spills['geometry'] = gpd.points_from_xy(spills['Longitude'], spills['Latitude'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "789eaccb",
   "metadata": {},
   "outputs": [],
   "source": [
    "spills_gdf = gpd.GeoDataFrame(spills, crs='EPSG:4326') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "d041a419",
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Ensure date columns are in datetime format\n",
    "spills_gdf['Date of Discovery'] = pd.to_datetime(spills_gdf['Date of Discovery'])\n",
    "spills_gdf['Initial Report Date'] = pd.to_datetime(spills_gdf['Initial Report Date'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "3106f3c5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# keep only records where year is between 2017 and 2023 (inclusive)\n",
    "spills_gdf = spills_gdf[\n",
    "    (spills_gdf['Date of Discovery'].dt.year >= 2017) &\n",
    "    (spills_gdf['Date of Discovery'].dt.year < 2024) &\n",
    "    (spills_gdf['Initial Report Date'].dt.year >= 2017) &\n",
    "    (spills_gdf['Initial Report Date'].dt.year < 2024)\n",
    "]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "139de8e5",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate Historical vs. Recent Spills\n",
    "historical_spills = spills_gdf[spills_gdf['Spill Type'] == 'Historical']\n",
    "recent_spills = spills_gdf[spills_gdf['Spill Type'] == 'Recent']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "853ce353",
   "metadata": {},
   "outputs": [],
   "source": [
    "# only use the top 3 counties by number of spills\n",
    "top_counties = spills_gdf['county'].value_counts().nlargest(3).index.tolist()\n",
    "# filter spills_gdf to only include top counties\n",
    "spills_gdf = spills_gdf[spills_gdf['county'].isin(top_counties)]\n",
    "# filter historical spills to only include top counties\n",
    "historical_spills = historical_spills[historical_spills['county'].isin(top_counties)]\n",
    "# filter recent spills to only include top counties\n",
    "recent_spills = recent_spills[recent_spills['county'].isin(top_counties)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "1327b4ea",
   "metadata": {},
   "outputs": [],
   "source": [
    "import statsmodels.formula.api as smf\n",
    "\n",
    "# Create 'Period' column\n",
    "spills_gdf['Period'] = spills_gdf['Report Year'].apply(lambda x: 'Before 2020' if x < 2020 else '2020 and After')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "2ccf5c3a",
   "metadata": {},
   "outputs": [],
   "source": [
    "spills_gdf = spills_gdf.rename(columns={\n",
    "    'Report Delay (Days)': 'report_delay',\n",
    "    'Spill Type': 'spill_type'\n",
    "})\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1da721cd",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "1b9a3766",
   "metadata": {},
   "source": [
    "# OLS Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "1b722ad7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:           report_delay   R-squared:                       0.001\n",
      "Model:                            OLS   Adj. R-squared:                  0.001\n",
      "Method:                 Least Squares   F-statistic:                     2.895\n",
      "Date:                Wed, 02 Jul 2025   Prob (F-statistic):             0.0339\n",
      "Time:                        12:54:40   Log-Likelihood:                -40805.\n",
      "No. Observations:                9087   AIC:                         8.162e+04\n",
      "Df Residuals:                    9083   BIC:                         8.165e+04\n",
      "Df Model:                           3                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "====================================================================================================================\n",
      "                                                       coef    std err          t      P>|t|      [0.025      0.975]\n",
      "--------------------------------------------------------------------------------------------------------------------\n",
      "Intercept                                            3.6818      0.454      8.117      0.000       2.793       4.571\n",
      "C(spill_type)[T.Recent]                             -1.1410      0.589     -1.938      0.053      -2.295       0.013\n",
      "C(Period)[T.Before 2020]                            -0.4169      0.773     -0.539      0.590      -1.932       1.098\n",
      "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]    -0.2081      0.969     -0.215      0.830      -2.107       1.691\n",
      "==============================================================================\n",
      "Omnibus:                    24587.315   Durbin-Watson:                   1.954\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):       1077831771.927\n",
      "Skew:                          33.014   Prob(JB):                         0.00\n",
      "Kurtosis:                    1688.924   Cond. No.                         7.42\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.formula.api as smf\n",
    "\n",
    "model = smf.ols(\"report_delay ~ C(spill_type) * C(Period)\", data=spills_gdf).fit()\n",
    "print(model.summary())\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0cef322c",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ee48e488",
   "metadata": {},
   "source": [
    "# Negative Binomial Regression"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "813a6d9c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 Generalized Linear Model Regression Results                  \n",
      "==============================================================================\n",
      "Dep. Variable:           report_delay   No. Observations:                 9087\n",
      "Model:                            GLM   Df Residuals:                     9083\n",
      "Model Family:        NegativeBinomial   Df Model:                            3\n",
      "Link Function:                    Log   Scale:                          1.0000\n",
      "Method:                          IRLS   Log-Likelihood:                -19592.\n",
      "Date:                Wed, 02 Jul 2025   Deviance:                       19541.\n",
      "Time:                        12:54:40   Pearson chi2:                 3.97e+05\n",
      "No. Iterations:                     8   Pseudo R-squ. (CS):            0.04240\n",
      "Covariance Type:            nonrobust                                         \n",
      "====================================================================================================================\n",
      "                                                       coef    std err          z      P>|z|      [0.025      0.975]\n",
      "--------------------------------------------------------------------------------------------------------------------\n",
      "Intercept                                            1.3034      0.024     54.986      0.000       1.257       1.350\n",
      "C(spill_type)[T.Recent]                             -0.3709      0.031    -11.830      0.000      -0.432      -0.309\n",
      "C(Period)[T.Before 2020]                            -0.1202      0.041     -2.949      0.003      -0.200      -0.040\n",
      "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]    -0.1622      0.052     -3.101      0.002      -0.265      -0.060\n",
      "====================================================================================================================\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dadams/miniconda3/envs/spatial_env2/lib/python3.10/site-packages/statsmodels/genmod/families/family.py:1367: ValueWarning: Negative binomial dispersion parameter alpha not set. Using default value alpha=1.0.\n",
      "  warnings.warn(\"Negative binomial dispersion parameter alpha not \"\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.formula.api as smf\n",
    "import statsmodels.api as sm\n",
    "\n",
    "# Make sure spill_type and Period are clean, and report_delay is numeric\n",
    "spills_gdf = spills_gdf.rename(columns={\n",
    "    'Report Delay (Days)': 'report_delay',\n",
    "    'Spill Type': 'spill_type'\n",
    "})\n",
    "\n",
    "# Fit the Negative Binomial model with interaction\n",
    "nb_model = smf.glm(\n",
    "    formula=\"report_delay ~ C(spill_type) * C(Period)\",\n",
    "    data=spills_gdf,\n",
    "    family=sm.families.NegativeBinomial()\n",
    ").fit()\n",
    "\n",
    "# View the results\n",
    "print(nb_model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "4343c273",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
       "columns": [
        {
         "name": "index",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "0",
         "rawType": "float64",
         "type": "float"
        }
       ],
       "ref": "78c37b6f-ac8c-4540-88a3-b6745c911c6b",
       "rows": [
        [
         "Intercept",
         "3.681838267786129"
        ],
        [
         "C(spill_type)[T.Recent]",
         "0.6900928211391695"
        ],
        [
         "C(Period)[T.Before 2020]",
         "0.8867658690823642"
        ],
        [
         "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]",
         "0.8502810395791129"
        ]
       ],
       "shape": {
        "columns": 1,
        "rows": 4
       }
      },
      "text/plain": [
       "Intercept                                           3.681838\n",
       "C(spill_type)[T.Recent]                             0.690093\n",
       "C(Period)[T.Before 2020]                            0.886766\n",
       "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]    0.850281\n",
       "dtype: float64"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "np.exp(nb_model.params)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "f09f08d0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 56792 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import pandas as pd\n",
    "\n",
    "# Predicted delays by group\n",
    "group_labels = [\n",
    "    \"Historical, After 2021\",\n",
    "    \"Recent, After 2021\",\n",
    "    \"Historical, Before 2021\",\n",
    "    \"Recent, Before 2021\"\n",
    "]\n",
    "\n",
    "# From your exponentiated results\n",
    "predicted_delays = [\n",
    "    13.53,                         # Intercept only\n",
    "    13.53 * 0.216,                 # Intercept * Recent\n",
    "    13.53 * 2.198,                 # Intercept * Before 2021\n",
    "    13.53 * 0.216 * 2.198 * 0.732  # Intercept * Recent * Before 2021 * Interaction\n",
    "]\n",
    "\n",
    "# Create DataFrame for plotting\n",
    "df_plot = pd.DataFrame({\n",
    "    \"Group\": group_labels,\n",
    "    \"Predicted Delay (Days)\": predicted_delays\n",
    "})\n",
    "\n",
    "# Plot\n",
    "plt.figure(figsize=(10, 6))\n",
    "bars = plt.bar(df_plot[\"Group\"], df_plot[\"Predicted Delay (Days)\"], alpha=0.8)\n",
    "plt.title(\"Predicted Report Delay by Spill Type and Period\", fontsize=14)\n",
    "plt.ylabel(\"Predicted Delay (Days)\", fontsize=12)\n",
    "plt.xticks(rotation=15, ha='right')\n",
    "plt.grid(axis='y', linestyle='--', alpha=0.7)\n",
    "plt.tight_layout()\n",
    "\n",
    "# Annotate bars\n",
    "for bar in bars:\n",
    "    height = bar.get_height()\n",
    "    plt.annotate(f\"{height:.2f}\",\n",
    "                 xy=(bar.get_x() + bar.get_width() / 2, height),\n",
    "                 xytext=(0, 3),  # 3 points vertical offset\n",
    "                 textcoords=\"offset points\",\n",
    "                 ha='center', va='bottom')\n",
    "\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "daa4b710",
   "metadata": {},
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "3238be11",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "f6c679b7",
   "metadata": {},
   "source": [
    "# Interrupted Time Series"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "fc8dc178",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:           report_delay   R-squared:                       0.016\n",
      "Model:                            OLS   Adj. R-squared:                 -0.021\n",
      "Method:                 Least Squares   F-statistic:                    0.4256\n",
      "Date:                Wed, 02 Jul 2025   Prob (F-statistic):              0.735\n",
      "Time:                        12:54:41   Log-Likelihood:                -211.00\n",
      "No. Observations:                  84   AIC:                             430.0\n",
      "Df Residuals:                      80   BIC:                             439.7\n",
      "Df Model:                           3                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "Intercept      2.2068      0.867      2.546      0.013       0.482       3.932\n",
      "time           0.0175      0.032      0.551      0.584      -0.046       0.081\n",
      "post_2021     -0.3117      3.266     -0.095      0.924      -6.811       6.187\n",
      "time_post      0.0026      0.057      0.045      0.964      -0.111       0.116\n",
      "==============================================================================\n",
      "Omnibus:                       66.933   Durbin-Watson:                   2.457\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              320.558\n",
      "Skew:                           2.631   Prob(JB):                     2.46e-70\n",
      "Kurtosis:                      10.994   Cond. No.                         625.\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 134796 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import statsmodels.formula.api as smf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 1. Convert to monthly time series\n",
    "spills_gdf['report_month'] = spills_gdf['Initial Report Date'].dt.to_period('M')\n",
    "monthly = spills_gdf.groupby('report_month')['report_delay'].mean().reset_index()\n",
    "monthly['report_month'] = monthly['report_month'].dt.to_timestamp()\n",
    "\n",
    "# 2. Create ITS variables\n",
    "monthly['time'] = (monthly['report_month'] - monthly['report_month'].min()).dt.days // 30\n",
    "monthly['post_2021'] = (monthly['report_month'] >= pd.Timestamp('2021-01-01')).astype(int)\n",
    "monthly['time_post'] = monthly['time'] * monthly['post_2021']\n",
    "\n",
    "# 3. Run the ITS model\n",
    "its_model = smf.ols(\"report_delay ~ time + post_2021 + time_post\", data=monthly).fit()\n",
    "print(its_model.summary())\n",
    "\n",
    "# 4. Predict and plot\n",
    "monthly['predicted'] = its_model.predict(monthly)\n",
    "\n",
    "plt.figure(figsize=(12, 6))\n",
    "plt.plot(monthly['report_month'], monthly['report_delay'], marker='o', label='Observed', alpha=0.6)\n",
    "plt.plot(monthly['report_month'], monthly['predicted'], linestyle='--', color='red', label='Predicted (ITS)')\n",
    "plt.axvline(x=pd.Timestamp('2021-01-01'), color='black', linestyle='--', label='Policy Change (2021)')\n",
    "plt.title('Interrupted Time Series: Monthly Report Delay')\n",
    "plt.xlabel('Month')\n",
    "plt.ylabel('Mean Report Delay (Days)')\n",
    "plt.legend()\n",
    "plt.grid(True)\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "6ae433ec",
   "metadata": {},
   "source": [
    "## ITS by Spill Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "bfd7557f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "=== ITS Model Summary for Historical Spills ===\n",
      "\n",
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:           report_delay   R-squared:                       0.018\n",
      "Model:                            OLS   Adj. R-squared:                 -0.019\n",
      "Method:                 Least Squares   F-statistic:                    0.4853\n",
      "Date:                Wed, 02 Jul 2025   Prob (F-statistic):              0.693\n",
      "Time:                        12:54:41   Log-Likelihood:                -237.68\n",
      "No. Observations:                  84   AIC:                             483.4\n",
      "Df Residuals:                      80   BIC:                             493.1\n",
      "Df Model:                           3                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "Intercept      3.6465      1.191      3.062      0.003       1.277       6.016\n",
      "time          -0.0298      0.044     -0.684      0.496      -0.117       0.057\n",
      "post_2021     -1.7128      4.487     -0.382      0.704     -10.642       7.216\n",
      "time_post      0.0578      0.078      0.740      0.462      -0.098       0.213\n",
      "==============================================================================\n",
      "Omnibus:                       64.999   Durbin-Watson:                   2.155\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):              304.862\n",
      "Skew:                           2.539   Prob(JB):                     6.31e-67\n",
      "Kurtosis:                      10.831   Cond. No.                         625.\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
      "\n",
      "=== ITS Model Summary for Recent Spills ===\n",
      "\n",
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:           report_delay   R-squared:                       0.008\n",
      "Model:                            OLS   Adj. R-squared:                 -0.029\n",
      "Method:                 Least Squares   F-statistic:                    0.2274\n",
      "Date:                Wed, 02 Jul 2025   Prob (F-statistic):              0.877\n",
      "Time:                        12:54:41   Log-Likelihood:                -238.63\n",
      "No. Observations:                  84   AIC:                             485.3\n",
      "Df Residuals:                      80   BIC:                             495.0\n",
      "Df Model:                           3                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "==============================================================================\n",
      "                 coef    std err          t      P>|t|      [0.025      0.975]\n",
      "------------------------------------------------------------------------------\n",
      "Intercept      1.6599      1.204      1.378      0.172      -0.737       4.056\n",
      "time           0.0285      0.044      0.645      0.521      -0.059       0.116\n",
      "post_2021      0.0436      4.538      0.010      0.992      -8.987       9.074\n",
      "time_post     -0.0125      0.079     -0.158      0.875      -0.170       0.145\n",
      "==============================================================================\n",
      "Omnibus:                      118.520   Durbin-Watson:                   2.217\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             2530.024\n",
      "Skew:                           4.795   Prob(JB):                         0.00\n",
      "Kurtosis:                      28.118   Cond. No.                         625.\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import statsmodels.formula.api as smf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Group by month and spill type\n",
    "spills_gdf['report_month'] = spills_gdf['Initial Report Date'].dt.to_period('M')\n",
    "monthly_by_type = spills_gdf.groupby(['report_month', 'spill_type'])['report_delay'].mean().reset_index()\n",
    "monthly_by_type['report_month'] = monthly_by_type['report_month'].dt.to_timestamp()\n",
    "\n",
    "# ITS variables\n",
    "monthly_by_type['time'] = (monthly_by_type['report_month'] - monthly_by_type['report_month'].min()).dt.days // 30\n",
    "monthly_by_type['post_2021'] = (monthly_by_type['report_month'] >= pd.Timestamp('2021-01-01')).astype(int)\n",
    "monthly_by_type['time_post'] = monthly_by_type['time'] * monthly_by_type['post_2021']\n",
    "\n",
    "# Run ITS models and print summaries\n",
    "its_results_by_type = {}\n",
    "for stype in monthly_by_type['spill_type'].unique():\n",
    "    df = monthly_by_type[monthly_by_type['spill_type'] == stype].copy()\n",
    "    model = smf.ols(\"report_delay ~ time + post_2021 + time_post\", data=df).fit()\n",
    "    df['predicted'] = model.predict(df)\n",
    "    its_results_by_type[stype] = (model, df)\n",
    "    \n",
    "    print(f\"\\n=== ITS Model Summary for {stype} Spills ===\\n\")\n",
    "    print(model.summary())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d790d469",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 212356 characters]",
      "text/plain": [
       "<Figure size 1200x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import statsmodels.formula.api as smf\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# 1. Group by month and spill type\n",
    "spills_gdf['report_month'] = spills_gdf['Initial Report Date'].dt.to_period('M')\n",
    "monthly_by_type = spills_gdf.groupby(['report_month', 'spill_type'])['report_delay'].mean().reset_index()\n",
    "monthly_by_type['report_month'] = monthly_by_type['report_month'].dt.to_timestamp()\n",
    "\n",
    "# 2. Create ITS variables\n",
    "monthly_by_type['time'] = (monthly_by_type['report_month'] - monthly_by_type['report_month'].min()).dt.days // 30\n",
    "monthly_by_type['post_2021'] = (monthly_by_type['report_month'] >= pd.Timestamp('2021-01-01')).astype(int)\n",
    "monthly_by_type['time_post'] = monthly_by_type['time'] * monthly_by_type['post_2021']\n",
    "\n",
    "# 3. Run ITS models separately by spill type\n",
    "its_results_by_type = {}\n",
    "for stype in monthly_by_type['spill_type'].unique():\n",
    "    df = monthly_by_type[monthly_by_type['spill_type'] == stype].copy()\n",
    "    model = smf.ols(\"report_delay ~ time + post_2021 + time_post\", data=df).fit()\n",
    "    df['predicted'] = model.predict(df)\n",
    "    its_results_by_type[stype] = (model, df)\n",
    "\n",
    "# 4. Plot side-by-side\n",
    "fig, axs = plt.subplots(2, 1, figsize=(12, 10), sharex=True)\n",
    "\n",
    "for ax, (stype, (model, df)) in zip(axs, its_results_by_type.items()):\n",
    "    ax.plot(df['report_month'], df['report_delay'], marker='o', label='Observed', alpha=0.6)\n",
    "    ax.plot(df['report_month'], df['predicted'], linestyle='--', color='red', label='Predicted (ITS)')\n",
    "    ax.axvline(x=pd.Timestamp('2021-01-01'), color='black', linestyle='--', label='Policy Change (2021)')\n",
    "    ax.set_title(f\"ITS Model for {stype} Spills\")\n",
    "    ax.set_ylabel(\"Mean Report Delay (Days)\")\n",
    "    ax.legend()\n",
    "    ax.grid(True)\n",
    "\n",
    "axs[1].set_xlabel(\"Month\")\n",
    "plt.tight_layout()\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "efc4708c",
   "metadata": {},
   "source": [
    "---"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "610abfb4",
   "metadata": {},
   "source": [
    "# Conclusion: Comparative Summary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "546334b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_regression_results(model, model_name):\n",
    "    coefs = model.params\n",
    "    ses = model.bse\n",
    "    pvals = model.pvalues\n",
    "\n",
    "    df = pd.DataFrame({\n",
    "        'Coefficient': coefs,\n",
    "        'Std. Error': ses,\n",
    "        'P-Value': pvals\n",
    "    })\n",
    "    df['Model'] = model_name\n",
    "    return df.reset_index().rename(columns={'index': 'Term'})\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "a09b247e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
       "columns": [
        {
         "name": "Term",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "ITS (Monthly, All Spills)",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "ITS (Monthly, Historical)",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "ITS (Monthly, Recent)",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Negative Binomial",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "OLS (Interaction)",
         "rawType": "float64",
         "type": "float"
        }
       ],
       "ref": "6c82bb30-a5db-4b21-9109-e8b281128edb",
       "rows": [
        [
         "C(Period)[T.Before 2020]",
         null,
         null,
         null,
         "-0.12",
         null
        ],
        [
         "C(spill_type)[T.Recent]",
         null,
         null,
         null,
         "-0.371",
         null
        ],
        [
         "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]",
         null,
         null,
         null,
         "-0.162",
         null
        ],
        [
         "Intercept",
         "2.207",
         "3.646",
         "1.66",
         "1.303",
         "1.66"
        ],
        [
         "post_2021",
         "-0.312",
         "-1.713",
         "0.044",
         null,
         "0.044"
        ],
        [
         "time",
         "0.017",
         "-0.03",
         "0.028",
         null,
         "0.028"
        ],
        [
         "time_post",
         "0.003",
         "0.058",
         "-0.013",
         null,
         "-0.013"
        ]
       ],
       "shape": {
        "columns": 5,
        "rows": 7
       }
      },
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Model</th>\n",
       "      <th>ITS (Monthly, All Spills)</th>\n",
       "      <th>ITS (Monthly, Historical)</th>\n",
       "      <th>ITS (Monthly, Recent)</th>\n",
       "      <th>Negative Binomial</th>\n",
       "      <th>OLS (Interaction)</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Term</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>C(Period)[T.Before 2020]</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.120</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C(spill_type)[T.Recent]</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.371</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>C(spill_type)[T.Recent]:C(Period)[T.Before 2020]</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.162</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Intercept</th>\n",
       "      <td>2.207</td>\n",
       "      <td>3.646</td>\n",
       "      <td>1.660</td>\n",
       "      <td>1.303</td>\n",
       "      <td>1.660</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>post_2021</th>\n",
       "      <td>-0.312</td>\n",
       "      <td>-1.713</td>\n",
       "      <td>0.044</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.044</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time</th>\n",
       "      <td>0.017</td>\n",
       "      <td>-0.030</td>\n",
       "      <td>0.028</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.028</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>time_post</th>\n",
       "      <td>0.003</td>\n",
       "      <td>0.058</td>\n",
       "      <td>-0.013</td>\n",
       "      <td>NaN</td>\n",
       "      <td>-0.013</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Model                                             ITS (Monthly, All Spills)  \\\n",
       "Term                                                                          \n",
       "C(Period)[T.Before 2020]                                                NaN   \n",
       "C(spill_type)[T.Recent]                                                 NaN   \n",
       "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]                        NaN   \n",
       "Intercept                                                             2.207   \n",
       "post_2021                                                            -0.312   \n",
       "time                                                                  0.017   \n",
       "time_post                                                             0.003   \n",
       "\n",
       "Model                                             ITS (Monthly, Historical)  \\\n",
       "Term                                                                          \n",
       "C(Period)[T.Before 2020]                                                NaN   \n",
       "C(spill_type)[T.Recent]                                                 NaN   \n",
       "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]                        NaN   \n",
       "Intercept                                                             3.646   \n",
       "post_2021                                                            -1.713   \n",
       "time                                                                 -0.030   \n",
       "time_post                                                             0.058   \n",
       "\n",
       "Model                                             ITS (Monthly, Recent)  \\\n",
       "Term                                                                      \n",
       "C(Period)[T.Before 2020]                                            NaN   \n",
       "C(spill_type)[T.Recent]                                             NaN   \n",
       "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]                    NaN   \n",
       "Intercept                                                         1.660   \n",
       "post_2021                                                         0.044   \n",
       "time                                                              0.028   \n",
       "time_post                                                        -0.013   \n",
       "\n",
       "Model                                             Negative Binomial  \\\n",
       "Term                                                                  \n",
       "C(Period)[T.Before 2020]                                     -0.120   \n",
       "C(spill_type)[T.Recent]                                      -0.371   \n",
       "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]             -0.162   \n",
       "Intercept                                                     1.303   \n",
       "post_2021                                                       NaN   \n",
       "time                                                            NaN   \n",
       "time_post                                                       NaN   \n",
       "\n",
       "Model                                             OLS (Interaction)  \n",
       "Term                                                                 \n",
       "C(Period)[T.Before 2020]                                        NaN  \n",
       "C(spill_type)[T.Recent]                                         NaN  \n",
       "C(spill_type)[T.Recent]:C(Period)[T.Before 2020]                NaN  \n",
       "Intercept                                                     1.660  \n",
       "post_2021                                                     0.044  \n",
       "time                                                          0.028  \n",
       "time_post                                                    -0.013  "
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Collect regression summaries\n",
    "results_frames = []\n",
    "\n",
    "# Add your models — use your actual variable names\n",
    "results_frames.append(extract_regression_results(model, 'OLS (Interaction)'))\n",
    "results_frames.append(extract_regression_results(nb_model, 'Negative Binomial'))\n",
    "results_frames.append(extract_regression_results(its_model, 'ITS (Monthly, All Spills)'))\n",
    "\n",
    "\n",
    "# ITS by spill type (e.g., 'Historical', 'Recent')\n",
    "for spill_type, (model, df) in its_results_by_type.items():\n",
    "    label = f\"ITS (Monthly, {spill_type})\"\n",
    "    results_frames.append(extract_regression_results(model, label))\n",
    "\n",
    "# Combine all\n",
    "regression_summary_table = pd.concat(results_frames, ignore_index=True)\n",
    "\n",
    "# Pivot just the coefficients for display\n",
    "regression_coef_table = regression_summary_table.pivot_table(\n",
    "    index='Term',\n",
    "    columns='Model',\n",
    "    values='Coefficient',\n",
    "    aggfunc='first'\n",
    ").round(3)\n",
    "\n",
    "regression_coef_table\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "spatial_env2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
