{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dadams/miniconda3/envs/spatial_env2/lib/python3.10/site-packages/pandas/io/sql.py:1725: SAWarning: Did not recognize type 'geometry' of column 'geometry'\n",
      "  self.meta.reflect(bind=self.con, only=[table_name], views=True)\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "import geopandas as gpd\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "import os\n",
    "\n",
    "# Database connection details from zshrc environment variables\n",
    "#  use .env file for local development\n",
    "\n",
    "\n",
    "db_name = 'colorado_spills'\n",
    "user = os.getenv('DB_USER')\n",
    "password = os.getenv('DB_PASSWORD')\n",
    "host = os.getenv('DB_HOST')\n",
    "port = os.getenv('DB_PORT')\n",
    "\n",
    "\n",
    "# Create an engine to connect to the PostgreSQL database\n",
    "engine = create_engine(f'postgresql+psycopg2://{user}:{password}@{host}:{port}/{db_name}')\n",
    "\n",
    "# Read in the spills_with_demographics_geog as spills\n",
    "spills = pd.read_sql_table('spills_with_demographics_geog', engine)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use longitude and latitude to create a GeoDataFrame from spills data\n",
    "spills['geometry'] = gpd.points_from_xy(spills['Longitude'], spills['Latitude'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "spills_gdf = gpd.GeoDataFrame(spills, crs='EPSG:4326') \n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "# Ensure date columns are in datetime format\n",
    "spills_gdf['Date of Discovery'] = pd.to_datetime(spills_gdf['Date of Discovery'])\n",
    "spills_gdf['Initial Report Date'] = pd.to_datetime(spills_gdf['Initial Report Date'])\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "# drop 2024 data for date of discovery and initial report date\n",
    "spills_gdf = spills_gdf[spills_gdf['Date of Discovery'].dt.year < 2024]\n",
    "spills_gdf = spills_gdf[spills_gdf['Initial Report Date'].dt.year < 2024]\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Separate Historical vs. Recent Spills\n",
    "historical_spills = spills_gdf[spills_gdf['Spill Type'] == 'Historical']\n",
    "recent_spills = spills_gdf[spills_gdf['Spill Type'] == 'Recent']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "              count       mean         std  min  25%  50%  75%     max\n",
      "Spill Type                                                            \n",
      "Historical   5312.0  21.490023  205.615913  0.0  0.0  1.0  2.0  9261.0\n",
      "Recent      10678.0   3.964506   46.997749  0.0  0.0  1.0  2.0  2232.0\n"
     ]
    }
   ],
   "source": [
    "# Compare Reporting Delay\n",
    "report_delay_summary = spills_gdf.groupby('Spill Type')['Report Delay (Days)'].describe()\n",
    "print(report_delay_summary)\n",
    "\n",
    "# Ensure the output directory exists\n",
    "output_dir = 'analysis/output'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Save the report delay summary to a markdown file\n",
    "with open(os.path.join(output_dir, 'report_delay_summary.md'), 'w') as f:\n",
    "    f.write(\"# Report Delay Summary\\n\\n\")\n",
    "    f.write(report_delay_summary.to_markdown())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 32240 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Compare Spill Trends Over Time\n",
    "spills_by_year = spills_gdf.groupby(['Report Year', 'Spill Type']).size().unstack()\n",
    "ax = spills_by_year.plot(kind='bar', figsize=(12,6), title='Spills by Year and Type')\n",
    "ax.set_xlabel('Year')\n",
    "ax.set_ylabel('Number of Spills')\n",
    "ax.legend(title='Spill Type')\n",
    "\n",
    "# Ensure the output directory exists\n",
    "output_dir = 'analysis/output'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Save the plot\n",
    "plt.savefig(f'{output_dir}/spills_by_year_and_type.png')\n",
    "\n",
    "# Display the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 154692 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import geopandas as gpd\n",
    "\n",
    "# Ensure spills_gdf is already a GeoDataFrame with the necessary data\n",
    "# spills_gdf = gpd.GeoDataFrame(...)\n",
    "\n",
    "# Create a plot for the spatial distribution of spills\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Plot the GeoDataFrame with optimizations\n",
    "spills_gdf.plot(\n",
    "    ax=ax,\n",
    "    marker='o', \n",
    "    alpha=0.6,  # Adjust transparency for better visibility\n",
    "    column='Spill Type', \n",
    "    legend=True, \n",
    "    legend_kwds={'title': \"Spill Type\"},  # Correctly specify the legend title\n",
    "    cmap='viridis'  # Use a colormap for better visual distinction\n",
    ")\n",
    "\n",
    "# Set plot title and labels\n",
    "ax.set_title('Spatial Distribution of Historical vs. Recent Spills', fontsize=14)\n",
    "ax.set_xlabel('Longitude', fontsize=12)\n",
    "ax.set_ylabel('Latitude', fontsize=12)\n",
    "\n",
    "# Improve layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Display the plot\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 470096 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import matplotlib.pyplot as plt\n",
    "import contextily as ctx\n",
    "import numpy as np\n",
    "\n",
    "# Use your ORIGINAL plot code exactly as it is, just adding the basemap at the end\n",
    "# This ensures all your data points, colors and categories remain intact\n",
    "\n",
    "# Starting with your original code:\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Plot the GeoDataFrame with optimizations - YOUR ORIGINAL PLOTTING CODE\n",
    "spills_gdf.plot(\n",
    "    ax=ax,\n",
    "    marker='o',\n",
    "    alpha=0.6,  # Adjust transparency for better visibility\n",
    "    column='Spill Type',\n",
    "    legend=True,\n",
    "    legend_kwds={'title': \"Spill Type\"},  # Correctly specify the legend title\n",
    "    cmap='viridis'  # Use a colormap for better visual distinction\n",
    ")\n",
    "\n",
    "# Set plot title and labels\n",
    "ax.set_title('Spatial Distribution of Historical vs. Recent Spills in Colorado', fontsize=14)\n",
    "ax.set_xlabel('Longitude', fontsize=12)\n",
    "ax.set_ylabel('Latitude', fontsize=12)\n",
    "\n",
    "# Store the current axis limits to restore them after adding basemap\n",
    "xlim = ax.get_xlim()\n",
    "ylim = ax.get_ylim()\n",
    "\n",
    "# Before adding the basemap, convert the axes coordinates to Web Mercator\n",
    "# This is a crucial step that lets us keep your original points exactly as they are\n",
    "from pyproj import Transformer\n",
    "\n",
    "# Create transformer from lat/lon to Web Mercator\n",
    "transformer = Transformer.from_crs(\"EPSG:4326\", \"EPSG:3857\", always_xy=True)\n",
    "\n",
    "# Transform the limits\n",
    "west, south = transformer.transform(xlim[0], ylim[0])\n",
    "east, north = transformer.transform(xlim[1], ylim[1])\n",
    "\n",
    "# Add the basemap - this will be UNDER your existing points\n",
    "ctx.add_basemap(\n",
    "    ax,\n",
    "    crs=spills_gdf.crs,\n",
    "    source=ctx.providers.OpenStreetMap.Mapnik,\n",
    "    zoom=\"auto\"\n",
    ")\n",
    "\n",
    "# Restore original axis limits\n",
    "ax.set_xlim(xlim)\n",
    "ax.set_ylim(ylim)\n",
    "\n",
    "# Improve layout\n",
    "plt.tight_layout()\n",
    "\n",
    "# Save the plot to the analysis/output directory\n",
    "output_dir = 'analysis/output'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "plt.savefig(f'{output_dir}/spatial_distribution_spills.png')\n",
    "\n",
    "# Show the plot\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                 count       mean         std  min  25%  50%  75%     max\n",
      "Period                                                                   \n",
      "2020 and After  7177.0   6.924342  124.965839  0.0  0.0  1.0  1.0  9261.0\n",
      "Before 2020     8813.0  12.117554  124.705876  0.0  0.0  1.0  2.0  5681.0\n"
     ]
    }
   ],
   "source": [
    "# Add a column to indicate whether the spill occurred before or after 2020\n",
    "spills_gdf['Period'] = spills_gdf['Report Year'].apply(lambda x: 'Before 2020' if x < 2020 else '2020 and After')\n",
    "\n",
    "# Group by the new period column and calculate the mean response time\n",
    "response_time_summary = spills_gdf.groupby('Period')['Report Delay (Days)'].describe()\n",
    "print(response_time_summary)\n",
    "\n",
    "# Ensure the output directory exists\n",
    "output_dir = 'analysis/output'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Save the response time summary to a markdown file\n",
    "with open(os.path.join(output_dir, 'response_time_summary.md'), 'w') as f:\n",
    "    f.write(\"# Response Time Summary\\n\\n\")\n",
    "    f.write(response_time_summary.to_markdown())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "T-Statistic: 2.6161\n",
      "P-Value: 0.0089\n",
      "The difference in response time is statistically significant.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<Axes: title={'center': 'Report Delay (Days)'}, xlabel='Period'>"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "[encoded data, 34072 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# t-test for the difference in response time before and after 2020\n",
    "from scipy.stats import ttest_ind\n",
    "\n",
    "before_2020 = spills_gdf[spills_gdf['Period'] == 'Before 2020']['Report Delay (Days)']\n",
    "after_2020 = spills_gdf[spills_gdf['Period'] == '2020 and After']['Report Delay (Days)']\n",
    "\n",
    "t_stat, p_value = ttest_ind(before_2020, after_2020, equal_var=False)\n",
    "print(f'T-Statistic: {t_stat:.4f}')\n",
    "print(f'P-Value: {p_value:.4f}')\n",
    "\n",
    "# Check if the difference is statistically significant\n",
    "if p_value < 0.05:\n",
    "    print('The difference in response time is statistically significant.')\n",
    "else:\n",
    "    print('The difference in response time is not statistically significant.')\n",
    "    \n",
    "# Create a plot for the response time distribution before and after 2020\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Plot the response time distribution for each period\n",
    "spills_gdf.boxplot(column='Report Delay (Days)', by='Period', ax=ax)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: tabulate in /home/dadams/miniconda3/envs/spatial_env2/lib/python3.10/site-packages (0.9.0)\n",
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install tabulate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Historical Spills Report Delay Summary:\n",
      "count    5312.000000\n",
      "mean       21.490023\n",
      "std       205.615913\n",
      "min         0.000000\n",
      "25%         0.000000\n",
      "50%         1.000000\n",
      "75%         2.000000\n",
      "max      9261.000000\n",
      "Name: Report Delay (Days), dtype: float64\n",
      "\n",
      "Recent Spills Report Delay Summary:\n",
      "count    10678.000000\n",
      "mean         3.964506\n",
      "std         46.997749\n",
      "min          0.000000\n",
      "25%          0.000000\n",
      "50%          1.000000\n",
      "75%          2.000000\n",
      "max       2232.000000\n",
      "Name: Report Delay (Days), dtype: float64\n",
      "\n",
      "Historical Spills Response Time Summary:\n",
      "                 count       mean         std  min  25%  50%  75%     max\n",
      "Period                                                                   \n",
      "2020 and After  2705.0  13.533087  200.352339  0.0  0.0  0.0  1.0  9261.0\n",
      "Before 2020     2607.0  29.746068  210.659491  0.0  0.0  1.0  2.0  5681.0\n",
      "\n",
      "Recent Spills Response Time Summary:\n",
      "                 count      mean        std  min  25%  50%  75%     max\n",
      "Period                                                                 \n",
      "2020 and After  4472.0  2.926878  27.302062  0.0  0.0  1.0  1.0  1329.0\n",
      "Before 2020     6206.0  4.712214  57.116100  0.0  1.0  1.0  2.0  2232.0\n",
      "\n",
      "Historical Spills T-Statistic: 2.8723\n",
      "Historical Spills P-Value: 0.0041\n",
      "The difference in response time for historical spills is statistically significant.\n",
      "\n",
      "Recent Spills T-Statistic: 2.1457\n",
      "Recent Spills P-Value: 0.0319\n",
      "The difference in response time for recent spills is statistically significant.\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/dadams/miniconda3/envs/spatial_env2/lib/python3.10/site-packages/geopandas/geodataframe.py:1819: SettingWithCopyWarning: \n",
      "A value is trying to be set on a copy of a slice from a DataFrame.\n",
      "Try using .loc[row_indexer,col_indexer] = value instead\n",
      "\n",
      "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
      "  super().__setitem__(key, value)\n"
     ]
    }
   ],
   "source": [
    "import tabulate\n",
    "from scipy.stats import ttest_ind\n",
    "\n",
    "# Ensure the output directory exists\n",
    "output_dir = 'analysis/output'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Add a column to indicate whether the spill occurred before or after 2020 for historical and recent spills\n",
    "historical_spills['Period'] = historical_spills['Report Year'].apply(lambda x: 'Before 2020' if x < 2020 else '2020 and After')\n",
    "recent_spills['Period'] = recent_spills['Report Year'].apply(lambda x: 'Before 2020' if x < 2020 else '2020 and After')\n",
    "\n",
    "# Historical Spills Analysis\n",
    "historical_report_delay_summary = historical_spills['Report Delay (Days)'].describe()\n",
    "print(\"Historical Spills Report Delay Summary:\")\n",
    "print(historical_report_delay_summary)\n",
    "\n",
    "# Recent Spills Analysis\n",
    "recent_report_delay_summary = recent_spills['Report Delay (Days)'].describe()\n",
    "print(\"\\nRecent Spills Report Delay Summary:\")\n",
    "print(recent_report_delay_summary)\n",
    "\n",
    "# Historical Spills Response Time Summary\n",
    "historical_response_time_summary = historical_spills.groupby('Period')['Report Delay (Days)'].describe()\n",
    "print(\"\\nHistorical Spills Response Time Summary:\")\n",
    "print(historical_response_time_summary)\n",
    "\n",
    "# Recent Spills Response Time Summary\n",
    "recent_response_time_summary = recent_spills.groupby('Period')['Report Delay (Days)'].describe()\n",
    "print(\"\\nRecent Spills Response Time Summary:\")\n",
    "print(recent_response_time_summary)\n",
    "\n",
    "# t-test for the difference in response time before and after 2020 for historical spills\n",
    "historical_before_2020 = historical_spills[historical_spills['Period'] == 'Before 2020']['Report Delay (Days)']\n",
    "historical_after_2020 = historical_spills[historical_spills['Period'] == '2020 and After']['Report Delay (Days)']\n",
    "\n",
    "t_stat_historical, p_value_historical = ttest_ind(historical_before_2020, historical_after_2020, equal_var=False)\n",
    "print(f'\\nHistorical Spills T-Statistic: {t_stat_historical:.4f}')\n",
    "print(f'Historical Spills P-Value: {p_value_historical:.4f}')\n",
    "\n",
    "# Check if the difference is statistically significant for historical spills\n",
    "if p_value_historical < 0.05:\n",
    "    print('The difference in response time for historical spills is statistically significant.')\n",
    "else:\n",
    "    print('The difference in response time for historical spills is not statistically significant.')\n",
    "    \n",
    "# t-test for the difference in response time before and after 2020 for recent spills\n",
    "recent_before_2020 = recent_spills[recent_spills['Period'] == 'Before 2020']['Report Delay (Days)']\n",
    "recent_after_2020 = recent_spills[recent_spills['Period'] == '2020 and After']['Report Delay (Days)']\n",
    "\n",
    "t_stat_recent, p_value_recent = ttest_ind(recent_before_2020, recent_after_2020, equal_var=False)\n",
    "print(f'\\nRecent Spills T-Statistic: {t_stat_recent:.4f}')\n",
    "print(f'Recent Spills P-Value: {p_value_recent:.4f}')\n",
    "\n",
    "# Check if the difference is statistically significant for recent spills\n",
    "if p_value_recent < 0.05:\n",
    "    print('The difference in response time for recent spills is statistically significant.')\n",
    "else:\n",
    "    print('The difference in response time for recent spills is not statistically significant.')\n",
    "\n",
    "# Save the summaries and t-test results to a markdown file\n",
    "with open(os.path.join(output_dir, 'analysis_results.md'), 'w') as f:\n",
    "    f.write(\"# Analysis Results\\n\\n\")\n",
    "    \n",
    "    f.write(\"## Historical Spills Report Delay Summary:\\n\")\n",
    "    f.write(historical_report_delay_summary.to_markdown() + \"\\n\\n\")\n",
    "    \n",
    "    f.write(\"## Recent Spills Report Delay Summary:\\n\")\n",
    "    f.write(recent_report_delay_summary.to_markdown() + \"\\n\\n\")\n",
    "    \n",
    "    f.write(\"## Historical Spills Response Time Summary:\\n\")\n",
    "    f.write(historical_response_time_summary.to_markdown() + \"\\n\\n\")\n",
    "    \n",
    "    f.write(\"## Recent Spills Response Time Summary:\\n\")\n",
    "    f.write(recent_response_time_summary.to_markdown() + \"\\n\\n\")\n",
    "    \n",
    "    f.write(\"## T-Test Results for Historical Spills\\n\")\n",
    "    f.write(f\"T-Statistic: {t_stat_historical:.4f}\\n\")\n",
    "    f.write(f\"P-Value: {p_value_historical:.4f}\\n\")\n",
    "    if p_value_historical < 0.05:\n",
    "        f.write('The difference in response time for historical spills is statistically significant.\\n')\n",
    "    else:\n",
    "        f.write('The difference in response time for historical spills is not statistically significant.\\n')\n",
    "    \n",
    "    f.write(\"\\n## T-Test Results for Recent Spills\\n\")\n",
    "    f.write(f\"T-Statistic: {t_stat_recent:.4f}\\n\")\n",
    "    f.write(f\"P-Value: {p_value_recent:.4f}\\n\")\n",
    "    if p_value_recent < 0.05:\n",
    "        f.write('The difference in response time for recent spills is statistically significant.\\n')\n",
    "    else:\n",
    "        f.write('The difference in response time for recent spills is not statistically significant.\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 45116 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 44908 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import os\n",
    "\n",
    "# Ensure the output directory exists\n",
    "output_dir = 'analysis/output'\n",
    "os.makedirs(output_dir, exist_ok=True)\n",
    "\n",
    "# Save the t-test results to a markdown file\n",
    "with open(os.path.join(output_dir, 't_test_results.md'), 'w') as f:\n",
    "    f.write(f\"## T-Test Results for Historical Spills\\n\")\n",
    "    f.write(f\"T-Statistic: {t_stat_historical:.4f}\\n\")\n",
    "    f.write(f\"P-Value: {p_value_historical:.4f}\\n\")\n",
    "    if p_value_historical < 0.05:\n",
    "        f.write('The difference in response time for historical spills is statistically significant.\\n')\n",
    "    else:\n",
    "        f.write('The difference in response time for historical spills is not statistically significant.\\n')\n",
    "    \n",
    "    f.write(f\"\\n## T-Test Results for Recent Spills\\n\")\n",
    "    f.write(f\"T-Statistic: {t_stat_recent:.4f}\\n\")\n",
    "    f.write(f\"P-Value: {p_value_recent:.4f}\\n\")\n",
    "    if p_value_recent < 0.05:\n",
    "        f.write('The difference in response time for recent spills is statistically significant.\\n')\n",
    "    else:\n",
    "        f.write('The difference in response time for recent spills is not statistically significant.\\n')\n",
    "\n",
    "# Create and save the plots\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "historical_spills.boxplot(column='Report Delay (Days)', by='Period', ax=ax)\n",
    "ax.set_title('Historical Spills Response Time Distribution Before and After 2020', fontsize=14)\n",
    "ax.set_xlabel('Period', fontsize=12)\n",
    "ax.set_ylabel('Report Delay (Days)', fontsize=12)\n",
    "plt.suptitle('')\n",
    "plt.tight_layout()\n",
    "plt.savefig(os.path.join(output_dir, 'historical_spills_response_time_distribution.png'))\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "recent_spills.boxplot(column='Report Delay (Days)', by='Period', ax=ax)\n",
    "ax.set_title('Recent Spills Response Time Distribution Before and After 2020', fontsize=14)\n",
    "ax.set_xlabel('Period', fontsize=12)\n",
    "ax.set_ylabel('Report Delay (Days)', fontsize=12)\n",
    "plt.suptitle('')\n",
    "plt.tight_layout()\n",
    "plt.savefig(os.path.join(output_dir, 'recent_spills_response_time_distribution.png'))\n",
    "\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Historical Spills T-Statistic: 2.8723\n",
      "Historical Spills P-Value: 0.0041\n",
      "The difference in response time for historical spills is statistically significant.\n",
      "\n",
      "Recent Spills T-Statistic: 2.1457\n",
      "Recent Spills P-Value: 0.0319\n",
      "The difference in response time for recent spills is statistically significant.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 37736 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# t-test for the difference in response time before and after 2020 for historical spills\n",
    "historical_before_2020 = historical_spills[historical_spills['Period'] == 'Before 2020']['Report Delay (Days)']\n",
    "historical_after_2020 = historical_spills[historical_spills['Period'] == '2020 and After']['Report Delay (Days)']\n",
    "\n",
    "t_stat, p_value = ttest_ind(historical_before_2020, historical_after_2020, equal_var=False)\n",
    "print(f'\\nHistorical Spills T-Statistic: {t_stat:.4f}')\n",
    "print(f'Historical Spills P-Value: {p_value:.4f}')\n",
    "\n",
    "# Check if the difference is statistically significant for historical spills\n",
    "if p_value < 0.05:\n",
    "    print('The difference in response time for historical spills is statistically significant.')\n",
    "else:\n",
    "    print('The difference in response time for historical spills is not statistically significant.')\n",
    "    \n",
    "# t-test for the difference in response time before and after 2020 for recent spills\n",
    "recent_before_2020 = recent_spills[recent_spills['Period'] == 'Before 2020']['Report Delay (Days)']\n",
    "recent_after_2020 = recent_spills[recent_spills['Period'] == '2020 and After']['Report Delay (Days)']\n",
    "\n",
    "t_stat, p_value = ttest_ind(recent_before_2020, recent_after_2020, equal_var=False)\n",
    "print(f'\\nRecent Spills T-Statistic: {t_stat:.4f}')\n",
    "print(f'Recent Spills P-Value: {p_value:.4f}')\n",
    "\n",
    "# Check if the difference is statistically significant for recent spills\n",
    "if p_value < 0.05:\n",
    "    print('The difference in response time for recent spills is statistically significant.')\n",
    "else:\n",
    "    print('The difference in response time for recent spills is not statistically significant.')\n",
    "    \n",
    "# Create a plot for the response time distribution before and after 2020 for historical and recent spills\n",
    "fig, ax = plt.subplots(figsize=(10, 6))\n",
    "\n",
    "# Plot the response time distribution for historical spills\n",
    "historical_spills.boxplot(column='Report Delay (Days)', by='Period', ax=ax, positions=[1, 2])\n",
    "\n",
    "# Plot the response time distribution for recent spills\n",
    "recent_spills.boxplot(column='Report Delay (Days)', by='Period', ax=ax, positions=[4, 5])\n",
    "\n",
    "# Save the plot to the /output directory\n",
    "plt.savefig('analysis/output/response_time_distribution.png')\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "spatial_env2",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
