{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.microsoft.datawrangler.viewer.v0+json": {
       "columns": [
        {
         "name": "index",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "Document #",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "Report",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Operator",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Operator #",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "Tracking #",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "Initial Report Date",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Date of Discovery",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Spill Type",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Qtr Qtr",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Section",
         "rawType": "int64",
         "type": "integer"
        },
        {
         "name": "Township",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "range",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "meridian",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Latitude",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Longitude",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Municipality",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "county",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Facility Type",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Facility ID",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "API County Code",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "API Sequence Number",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Spilled outside of berms",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "More than five barrels spilled",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Oil Spill Volume",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Condensate Spill Volume",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Flow Back Spill Volume",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Produced Water Spill Volume",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "E&P Waste Spill Volume",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Other Waste",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Drilling Fluid Spill Volume",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Current Land Use",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Other Land Use",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Weather Conditions",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Surface Owner",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Surface Owner Other",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Waters of the State",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Residence / Occupied Structure",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "livestock",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Public Byway",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Water Supply Area",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Spill Description",
         "rawType": "object",
         "type": "string"
        },
        {
         "name": "Supplemental Report Date",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Oil BBLs Spilled",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Oil BBLs Recovered",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Oil Unknown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Condensate BBLs Spilled",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Condensate BBLs Recovered",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Condensate Unknown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Produced Water BBLs Spilled",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Produced Water BBLs Recovered",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Produced Water Unknown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Drilling Fluid BBLs Spilled",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Drilling Fluid BBLs Recovered",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Drilling Fluid Unknown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Flow Back Fluid BBLs Spilled",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Flow Back Fluid BBLs Recovered",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Flow Back Fluid Unkown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other E&P Waste BBLS Spilled",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other E&P Waste BBLS Recovered",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other E&P Waste Unknown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other E&P Waste",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Spill Contained within Berm",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Emergency Pit Constructed",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "soil",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "groundwater",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Water",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Dry Drainage Feature",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Area Length",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Area Width",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Depth of Impact in Feet",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Depth of Impact in Inches",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Area Depth Determined",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Geology Description",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Depth to Groundwater",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Water wells in area",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Water Wells",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Water Wells None",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Water Near",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Water None",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Wetlands",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Wetlands None",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Springs",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Springs None",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Livestock Near",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Livestock None",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Occupied Buildings",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Occupied Buildings None",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Additional Spill Details",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Supplemental Report Date CA",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Human Error",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Equipment Failure",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Historical Unkown",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other Description",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Root Cause",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Preventative Measures",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Soil Excavated",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Offsite Disposal",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Onsite Treatment",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other Disposition",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Other Disposition Description",
         "rawType": "object",
         "type": "unknown"
        },
        {
         "name": "Ground Water Removed",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Surface Water Removed",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Corrective Actions Completed",
         "rawType": "bool",
         "type": "boolean"
        },
        {
         "name": "Approved Form 27",
         "rawType": "float64",
         "type": "float"
        },
        {
         "name": "Form 27 Project Number",
         "rawType": "float64",
         "type": "float"
        }
       ],
       "ref": "d0d92c1c-8fb1-46da-954b-64bc54899332",
       "rows": [
        [
         "0",
         "400811665",
         "I w/S",
         "CHEVRON USA INC",
         "16700",
         "400811665",
         "03/19/2015",
         "06/04/2012",
         "Historical",
         "nwne  ",
         "19",
         "2n    ",
         "102w   ",
         "6",
         "40.132395",
         "-108.882137",
         null,
         "RIO BLANCO     ",
         "FLOWLINE",
         null,
         "   ",
         "     ",
         "Y",
         "Y",
         ">0 and <1",
         "0",
         "0",
         ">=5 and <100",
         "0",
         ".37 bbl oil spilled, 47.84 bbl produced water spilled, 43bbl and .37bbl produced water recovered",
         "0",
         "NON-CROP LAND                      ",
         null,
         "85 degrees sunny                   ",
         "OTHER (SPECIFY)",
         "chevron surface",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "A leak occurred on a four inch check valve at Collection Station 12. Approximately 47. 8402 bbls of brine water and 0.3704 bbls of crude oil were released. The line was shut in immediately upon detection. Vaccum trucks removed all of the free fluid, estimated recovery is approximately 43 bbls of brine water and 0.3704 bbls of crude oil. The fluids that were picked up, were taken to the truck unloading facility at the Main Water Plant for recycling. Any impacted soil was excavated and taken to the landfarm.",
         "03/19/2015",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "47.0",
         "47.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         null,
         "0.0",
         "0.0",
         "1.0",
         "0.0",
         "0.0",
         "0.0",
         "200.0",
         null,
         null,
         null,
         "soil samples",
         "unknown",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "depths to pathways not provided by operator. ",
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         "False",
         "0.0",
         null
        ],
        [
         "1",
         "400811979",
         "I",
         "WHITING OIL & GAS CORPORATION",
         "96155",
         "400811979",
         "03/19/2015",
         "03/18/2015",
         "Recent",
         "SWSW  ",
         "33",
         "10N   ",
         "58W    ",
         "6",
         "40.789186",
         "-103.876525",
         null,
         "WELD           ",
         "WELL PAD",
         "431440.0",
         "   ",
         "     ",
         "Y",
         "N",
         "0",
         "0",
         ">=1 and <5",
         "0",
         "0",
         null,
         "0",
         "OTHER                              ",
         "Rangeland",
         "Partly cloudy, 50 F                ",
         "FEE",
         "Robert Rohn",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "Approximately 3 bbls of flowback water released from a frac tank at the Razor 33M pad during flowback operations. The water was contained with soil and the impacted soil was placed on a liner pending onsite treatment.",
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         "False",
         "0.0",
         null
        ],
        [
         "2",
         "400812102",
         "I w/S",
         "CHEVRON USA INC",
         "16700",
         "400812102",
         "03/19/2015",
         "07/11/2012",
         "Historical",
         "nese  ",
         "32",
         "2n    ",
         "102w   ",
         "6",
         "40.099592",
         "-108.861426",
         null,
         "RIO BLANCO     ",
         "OTHER",
         null,
         "   ",
         "     ",
         "Y",
         "Y",
         "0",
         "0",
         "0",
         ">=5 and <100",
         "0",
         "51 bbl of produced water was spilled , 50bbls were recovered",
         "0",
         "NON-CROP LAND                      ",
         null,
         "90 deg sunny                       ",
         "OTHER (SPECIFY)",
         "chevron surface",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "This spill is being entered by COGCC staff to document a historic spill. It is believed that proper notice was give by the operator at the time of spill, but it is not discoverable in the database.                                                                                                                                  A leak occurred on a 3/4 inch bleed valve nipple at the Main Water Plant's Solids Pump. Approximately 51 bbls of brine water was released. The line was shut in immediately upon detection. Vaccum trucks removed all of the free fluid, estimated recovery is approximately 50 bbls of brine water. The fluids that were picked up, were taken to the truck unloading facility at the Main Water Plant for recycling.",
         "03/19/2015",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "51.0",
         "50.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         null,
         "0.0",
         "0.0",
         "1.0",
         "0.0",
         "0.0",
         "0.0",
         "350.0",
         "1.0",
         "0.0",
         null,
         "depth unknown",
         "not reported. \"0\" reported where depths were not required at time of spill.",
         "0.0",
         "0.0",
         null,
         "1.0",
         null,
         "1.0",
         null,
         "1.0",
         null,
         "1.0",
         null,
         "1.0",
         null,
         "1.0",
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         "False",
         "0.0",
         null
        ],
        [
         "3",
         "400812258",
         "I",
         "KERR MCGEE OIL & GAS ONSHORE LP",
         "47120",
         "400812258",
         "03/20/2015",
         "03/19/2015",
         "Historical",
         "NENW  ",
         "13",
         "1N    ",
         "68W    ",
         "6",
         "40.057397",
         "-104.954297",
         null,
         "WELD           ",
         "TANK BATTERY",
         "327241.0",
         "   ",
         "     ",
         "Y",
         "N",
         "0",
         "Unknown",
         "0",
         "Unknown",
         "0",
         null,
         "0",
         "OTHER                              ",
         null,
         "50's, Sunny.                       ",
         "FEE",
         null,
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "0.0",
         "On March 19, 2015, historical impacts were discovered during plugging and abandonment activities at the UPRR 43 Pan Am O#2 production facility. Approximately 160 cubic yards of impacted material have been excavated and transported to the Front Range Regional Landfill located in Erie, Colorado for disposal. Excavation activities are on-going at this location.",
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         "False",
         "0.0",
         null
        ],
        [
         "4",
         "400812277",
         "I",
         "NOBLE ENERGY INC",
         "100322",
         "400812277",
         "03/20/2015",
         "03/18/2015",
         "Recent",
         "SENE  ",
         "28",
         "4N    ",
         "64W    ",
         "6",
         "40.286102",
         "-104.547566",
         null,
         "WELD           ",
         "FLOWLINE",
         "329948.0",
         "   ",
         "     ",
         "Y",
         "N",
         ">=1 and <5",
         "0",
         "0",
         "0",
         "0",
         null,
         "0",
         "NON-CROP LAND                      ",
         null,
         "40 rain                            ",
         "FEE",
         null,
         "0.0",
         "0.0",
         "1.0",
         "0.0",
         "0.0",
         "The flowline for the Nix 28-8i4 developed a leak in a dry canal.  The production equipment was shut in and an excavation will be performed.  Soil lab confirmation sampling will be performed by a 3rd party environmental consultant to identify the extent of the excavation.  The canal will be rebuilt with coordination from the ditch company.",
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         null,
         "False",
         "0.0",
         "2315486.0"
        ]
       ],
       "shape": {
        "columns": 106,
        "rows": 5
       }
      },
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document #</th>\n",
       "      <th>Report</th>\n",
       "      <th>Operator</th>\n",
       "      <th>Operator #</th>\n",
       "      <th>Tracking #</th>\n",
       "      <th>Initial Report Date</th>\n",
       "      <th>Date of Discovery</th>\n",
       "      <th>Spill Type</th>\n",
       "      <th>Qtr Qtr</th>\n",
       "      <th>Section</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil Excavated</th>\n",
       "      <th>Offsite Disposal</th>\n",
       "      <th>Onsite Treatment</th>\n",
       "      <th>Other Disposition</th>\n",
       "      <th>Other Disposition Description</th>\n",
       "      <th>Ground Water Removed</th>\n",
       "      <th>Surface Water Removed</th>\n",
       "      <th>Corrective Actions Completed</th>\n",
       "      <th>Approved Form 27</th>\n",
       "      <th>Form 27 Project Number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>400811665</td>\n",
       "      <td>I w/S</td>\n",
       "      <td>CHEVRON USA INC</td>\n",
       "      <td>16700</td>\n",
       "      <td>400811665</td>\n",
       "      <td>03/19/2015</td>\n",
       "      <td>06/04/2012</td>\n",
       "      <td>Historical</td>\n",
       "      <td>nwne</td>\n",
       "      <td>19</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>400811979</td>\n",
       "      <td>I</td>\n",
       "      <td>WHITING OIL &amp; GAS CORPORATION</td>\n",
       "      <td>96155</td>\n",
       "      <td>400811979</td>\n",
       "      <td>03/19/2015</td>\n",
       "      <td>03/18/2015</td>\n",
       "      <td>Recent</td>\n",
       "      <td>SWSW</td>\n",
       "      <td>33</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>400812102</td>\n",
       "      <td>I w/S</td>\n",
       "      <td>CHEVRON USA INC</td>\n",
       "      <td>16700</td>\n",
       "      <td>400812102</td>\n",
       "      <td>03/19/2015</td>\n",
       "      <td>07/11/2012</td>\n",
       "      <td>Historical</td>\n",
       "      <td>nese</td>\n",
       "      <td>32</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>400812258</td>\n",
       "      <td>I</td>\n",
       "      <td>KERR MCGEE OIL &amp; GAS ONSHORE LP</td>\n",
       "      <td>47120</td>\n",
       "      <td>400812258</td>\n",
       "      <td>03/20/2015</td>\n",
       "      <td>03/19/2015</td>\n",
       "      <td>Historical</td>\n",
       "      <td>NENW</td>\n",
       "      <td>13</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>400812277</td>\n",
       "      <td>I</td>\n",
       "      <td>NOBLE ENERGY INC</td>\n",
       "      <td>100322</td>\n",
       "      <td>400812277</td>\n",
       "      <td>03/20/2015</td>\n",
       "      <td>03/18/2015</td>\n",
       "      <td>Recent</td>\n",
       "      <td>SENE</td>\n",
       "      <td>28</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2315486.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Document # Report                         Operator  Operator #  Tracking #  \\\n",
       "0   400811665  I w/S                  CHEVRON USA INC       16700   400811665   \n",
       "1   400811979      I    WHITING OIL & GAS CORPORATION       96155   400811979   \n",
       "2   400812102  I w/S                  CHEVRON USA INC       16700   400812102   \n",
       "3   400812258      I  KERR MCGEE OIL & GAS ONSHORE LP       47120   400812258   \n",
       "4   400812277      I                 NOBLE ENERGY INC      100322   400812277   \n",
       "\n",
       "  Initial Report Date Date of Discovery  Spill Type Qtr Qtr  Section  ...  \\\n",
       "0          03/19/2015        06/04/2012  Historical  nwne         19  ...   \n",
       "1          03/19/2015        03/18/2015      Recent  SWSW         33  ...   \n",
       "2          03/19/2015        07/11/2012  Historical  nese         32  ...   \n",
       "3          03/20/2015        03/19/2015  Historical  NENW         13  ...   \n",
       "4          03/20/2015        03/18/2015      Recent  SENE         28  ...   \n",
       "\n",
       "  Soil Excavated Offsite Disposal Onsite Treatment  Other Disposition  \\\n",
       "0            NaN              NaN              NaN                NaN   \n",
       "1            NaN              NaN              NaN                NaN   \n",
       "2            NaN              NaN              NaN                NaN   \n",
       "3            NaN              NaN              NaN                NaN   \n",
       "4            NaN              NaN              NaN                NaN   \n",
       "\n",
       "   Other Disposition Description Ground Water Removed Surface Water Removed  \\\n",
       "0                           None                  NaN                   NaN   \n",
       "1                           None                  NaN                   NaN   \n",
       "2                           None                  NaN                   NaN   \n",
       "3                           None                  NaN                   NaN   \n",
       "4                           None                  NaN                   NaN   \n",
       "\n",
       "  Corrective Actions Completed  Approved Form 27 Form 27 Project Number  \n",
       "0                        False               0.0                    NaN  \n",
       "1                        False               0.0                    NaN  \n",
       "2                        False               0.0                    NaN  \n",
       "3                        False               0.0                    NaN  \n",
       "4                        False               0.0              2315486.0  \n",
       "\n",
       "[5 rows x 106 columns]"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "import geopandas as gpd\n",
    "from dotenv import load_dotenv\n",
    "load_dotenv()\n",
    "\n",
    "import os\n",
    "\n",
    "# Database connection details from zshrc environment variables\n",
    "db_name = 'colorado_spills'\n",
    "user = os.getenv('DB_USER')\n",
    "password = os.getenv('DB_PASSWORD')\n",
    "host = os.getenv('DB_HOST')\n",
    "port = os.getenv('DB_PORT')\n",
    "\n",
    "\n",
    "# Create an engine to connect to the PostgreSQL database\n",
    "engine = create_engine(f'postgresql+psycopg2://{user}:{password}@{host}:{port}/{db_name}')\n",
    "\n",
    "# Function to load data from a table\n",
    "def load_table(table_name):\n",
    "    query = f'SELECT * FROM {table_name}'\n",
    "    df = pd.read_sql(query, engine)\n",
    "    return df\n",
    "\n",
    "# Load the spills data\n",
    "spills_df = load_table('spills')\n",
    "spills_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Document #', 'Report', 'Operator', 'Operator #', 'Tracking #', 'Initial Report Date', 'Date of Discovery', 'Spill Type', 'Qtr Qtr', 'Section', 'Township', 'range', 'meridian', 'Latitude', 'Longitude', 'Municipality', 'county', 'Facility Type', 'Facility ID', 'API County Code', 'API Sequence Number', 'Spilled outside of berms', 'More than five barrels spilled', 'Oil Spill Volume', 'Condensate Spill Volume', 'Flow Back Spill Volume', 'Produced Water Spill Volume', 'E&P Waste Spill Volume', 'Other Waste', 'Drilling Fluid Spill Volume', 'Current Land Use', 'Other Land Use', 'Weather Conditions', 'Surface Owner', 'Surface Owner Other', 'Waters of the State', 'Residence / Occupied Structure', 'livestock', 'Public Byway', 'Surface Water Supply Area', 'Spill Description', 'Supplemental Report Date', 'Oil BBLs Spilled', 'Oil BBLs Recovered', 'Oil Unknown', 'Condensate BBLs Spilled', 'Condensate BBLs Recovered', 'Condensate Unknown', 'Produced Water BBLs Spilled', 'Produced Water BBLs Recovered', 'Produced Water Unknown', 'Drilling Fluid BBLs Spilled', 'Drilling Fluid BBLs Recovered', 'Drilling Fluid Unknown', 'Flow Back Fluid BBLs Spilled', 'Flow Back Fluid BBLs Recovered', 'Flow Back Fluid Unkown', 'Other E&P Waste BBLS Spilled', 'Other E&P Waste BBLS Recovered', 'Other E&P Waste Unknown', 'Other E&P Waste', 'Spill Contained within Berm', 'Emergency Pit Constructed', 'soil', 'groundwater', 'Surface Water', 'Dry Drainage Feature', 'Surface Area Length', 'Surface Area Width', 'Depth of Impact in Feet', 'Depth of Impact in Inches', 'Area Depth Determined', 'Geology Description', 'Depth to Groundwater', 'Water wells in area', 'Water Wells', 'Water Wells None', 'Surface Water Near', 'Surface Water None', 'Wetlands', 'Wetlands None', 'Springs', 'Springs None', 'Livestock Near', 'Livestock None', 'Occupied Buildings', 'Occupied Buildings None', 'Additional Spill Details', 'Supplemental Report Date CA', 'Human Error', 'Equipment Failure', 'Historical Unkown', 'Other', 'Other Description', 'Root Cause', 'Preventative Measures', 'Soil Excavated', 'Offsite Disposal', 'Onsite Treatment', 'Other Disposition', 'Other Disposition Description', 'Ground Water Removed', 'Surface Water Removed', 'Corrective Actions Completed', 'Approved Form 27', 'Form 27 Project Number']\n"
     ]
    }
   ],
   "source": [
    "# Assuming you have a DataFrame named df\n",
    "print(spills_df.columns.tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: us in /home/dadams/miniconda3/lib/python3.12/site-packages (3.2.0)\n",
      "Requirement already satisfied: jellyfish in /home/dadams/miniconda3/lib/python3.12/site-packages (from us) (1.2.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install  us"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Channels:\n",
      " - defaults\n",
      "Platform: linux-64\n",
      "Collecting package metadata (repodata.json): done\n",
      "Solving environment: done\n",
      "\n",
      "# All requested packages already installed.\n",
      "\n",
      "Requirement already satisfied: certifi in /home/dadams/miniconda3/lib/python3.12/site-packages (2025.8.3)\n"
     ]
    }
   ],
   "source": [
    "# In your terminal/conda prompt:\n",
    "!conda update ca-certificates\n",
    "# or if that doesn't work:\n",
    "!pip install --upgrade certifi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Census tract data has been added to spills_df.\n",
      "A plot of spills per tract has been generated.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 218940 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import us\n",
    "import requests\n",
    "from io import BytesIO\n",
    "from zipfile import ZipFile\n",
    "import os\n",
    "import shutil\n",
    "import urllib3\n",
    "\n",
    "# Disable SSL warnings\n",
    "urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)\n",
    "\n",
    "# Convert spills_df to a GeoDataFrame\n",
    "gdf = gpd.GeoDataFrame(\n",
    "    spills_df, \n",
    "    geometry=gpd.points_from_xy(spills_df.Longitude, spills_df.Latitude),\n",
    "    crs=\"EPSG:4326\"\n",
    ")\n",
    "\n",
    "# Get Colorado's FIPS code\n",
    "colorado_fips = us.states.CO.fips\n",
    "\n",
    "# Create a temporary directory for the shapefiles\n",
    "temp_dir = \"colorado_tracts_temp\"\n",
    "if not os.path.exists(temp_dir):\n",
    "    os.makedirs(temp_dir)\n",
    "\n",
    "# Your existing code with SSL verification disabled\n",
    "try:\n",
    "    # Download Colorado tract shapefile\n",
    "    url = f\"https://www2.census.gov/geo/tiger/TIGER2021/TRACT/tl_2021_{colorado_fips}_tract.zip\"\n",
    "    r = requests.get(url, verify=False)  # Add verify=False here\n",
    "    z = ZipFile(BytesIO(r.content))\n",
    "    z.extractall(temp_dir)\n",
    "\n",
    "    # Find the .shp file in the extracted contents\n",
    "    shp_file = next(f for f in os.listdir(temp_dir) if f.endswith('.shp'))\n",
    "    \n",
    "    # Read the shapefile\n",
    "    tracts = gpd.read_file(os.path.join(temp_dir, shp_file))\n",
    "    tracts = tracts.to_crs(gdf.crs)\n",
    "\n",
    "    # Perform spatial join\n",
    "    result = gpd.sjoin(gdf, tracts, how=\"left\", predicate=\"within\")\n",
    "\n",
    "    # Add census tract information to the original DataFrame\n",
    "    spills_df['GEOID'] = result['GEOID']\n",
    "    spills_df['TRACT_NAME'] = result['NAME']\n",
    "\n",
    "    # Count spills per census tract\n",
    "    spills_per_tract = spills_df.groupby('GEOID').size().reset_index(name='spill_count')\n",
    "    tracts_with_spills = tracts.merge(spills_per_tract, on='GEOID', how='left')\n",
    "    tracts_with_spills['spill_count'] = tracts_with_spills['spill_count'].fillna(0)\n",
    "\n",
    "    # Basic plot\n",
    "    tracts_with_spills.plot(column='spill_count', legend=True, figsize=(15, 10))\n",
    "\n",
    "finally:\n",
    "    # Clean up: remove the temporary directory\n",
    "    shutil.rmtree(temp_dir, ignore_errors=True)\n",
    "\n",
    "print(\"Census tract data has been added to spills_df.\")\n",
    "print(\"A plot of spills per tract has been generated.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: census in /home/dadams/miniconda3/lib/python3.12/site-packages (0.8.24)\n",
      "Requirement already satisfied: requests>=1.1.0 in /home/dadams/miniconda3/lib/python3.12/site-packages (from census) (2.32.3)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/dadams/miniconda3/lib/python3.12/site-packages (from requests>=1.1.0->census) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/dadams/miniconda3/lib/python3.12/site-packages (from requests>=1.1.0->census) (3.7)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/dadams/miniconda3/lib/python3.12/site-packages (from requests>=1.1.0->census) (2.2.3)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/dadams/miniconda3/lib/python3.12/site-packages (from requests>=1.1.0->census) (2025.8.3)\n"
     ]
    }
   ],
   "source": [
    "!pip install census"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         GEOID  total_population  percent_white  percent_hispanic  \\\n",
      "0  08001008354            4211.0      28.591783         23.557350   \n",
      "1  08001008401            5491.0      86.814788         14.059370   \n",
      "2  08001008402            4974.0      82.046643         14.053076   \n",
      "3  08001008526            7366.0      79.011675         16.806951   \n",
      "4  08001008538           10456.0      75.181714         30.174063   \n",
      "\n",
      "   median_household_income  percent_poverty  unemployment_rate  spill_count  \n",
      "0                 101250.0        10.757540           0.379957           33  \n",
      "1                  88286.0         5.918776           1.456930          107  \n",
      "2                 114276.0         4.121431           2.211500           71  \n",
      "3                  98283.0         3.801249           2.104263            1  \n",
      "4                 131231.0         4.150727           3.471691            9  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "import requests\n",
    "from census import Census\n",
    "from us import states\n",
    "\n",
    "import os\n",
    "\n",
    "# census API key from zshrc environment variable\n",
    "\n",
    "c = Census(os.getenv('CENSUS_API_KEY'))\n",
    "\n",
    "# Get unique GEOIDs from your spills data\n",
    "unique_geoids = spills_df['GEOID'].unique()\n",
    "\n",
    "# Function to get state and county FIPS codes from GEOID\n",
    "def get_state_county(geoid):\n",
    "    return geoid[:2], geoid[2:5]\n",
    "\n",
    "# Prepare list of unique state and county codes\n",
    "state_counties = list(set([get_state_county(geoid) for geoid in unique_geoids]))\n",
    "\n",
    "# List of variables to fetch (you can modify this based on your specific needs)\n",
    "variables = [\n",
    "    'B01003_001E',  # Total Population\n",
    "    'B02001_002E',  # White Population\n",
    "    'B03002_012E',  # Hispanic or Latino Population\n",
    "    'B19013_001E',  # Median Household Income\n",
    "    'B17001_002E',  # Population for whom poverty status is determined\n",
    "    'B23025_005E',  # Unemployed Population\n",
    "]\n",
    "\n",
    "# Fetch ACS data\n",
    "acs_data = []\n",
    "for state, county in state_counties:\n",
    "    acs_data.extend(c.acs5.state_county_tract(\n",
    "        variables,\n",
    "        state,\n",
    "        county,\n",
    "        Census.ALL,\n",
    "        year=2021\n",
    "    ))\n",
    "\n",
    "# Convert to DataFrame\n",
    "acs_df = pd.DataFrame(acs_data)\n",
    "\n",
    "# Rename columns\n",
    "acs_df = acs_df.rename(columns={\n",
    "    'B01003_001E': 'total_population',\n",
    "    'B02001_002E': 'white_population',\n",
    "    'B03002_012E': 'hispanic_population',\n",
    "    'B19013_001E': 'median_household_income',\n",
    "    'B17001_002E': 'poverty_population',\n",
    "    'B23025_005E': 'unemployed_population'\n",
    "})\n",
    "\n",
    "# Calculate percentages and rates\n",
    "acs_df['percent_white'] = acs_df['white_population'] / acs_df['total_population'] * 100\n",
    "acs_df['percent_hispanic'] = acs_df['hispanic_population'] / acs_df['total_population'] * 100\n",
    "acs_df['percent_poverty'] = acs_df['poverty_population'] / acs_df['total_population'] * 100\n",
    "acs_df['unemployment_rate'] = acs_df['unemployed_population'] / acs_df['total_population'] * 100\n",
    "\n",
    "# Create GEOID column\n",
    "acs_df['GEOID'] = acs_df['state'] + acs_df['county'] + acs_df['tract']\n",
    "\n",
    "# Merge ACS data with spills data\n",
    "spills_ej_df = spills_df.merge(acs_df, on='GEOID', how='left')\n",
    "\n",
    "# Aggregate spills data by census tract\n",
    "spills_by_tract = spills_ej_df.groupby('GEOID').agg({\n",
    "    'total_population': 'first',\n",
    "    'percent_white': 'first',\n",
    "    'percent_hispanic': 'first',\n",
    "    'median_household_income': 'first',\n",
    "    'percent_poverty': 'first',\n",
    "    'unemployment_rate': 'first',\n",
    "    'Document #': 'count'  # Assuming this is a unique identifier for each spill\n",
    "}).reset_index()\n",
    "\n",
    "spills_by_tract = spills_by_tract.rename(columns={'Document #': 'spill_count'})\n",
    "\n",
    "print(spills_by_tract.head())\n",
    "\n",
    "# You can now use spills_by_tract for further analysis and visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158, 8)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spills_by_tract.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original spills_df shape: (17820, 108)\n",
      "spills_with_demographics shape: (17820, 118)\n",
      "   Document # Report                         Operator  Operator #  Tracking #  \\\n",
      "0   400811665  I w/S                  CHEVRON USA INC       16700   400811665   \n",
      "1   400811979      I    WHITING OIL & GAS CORPORATION       96155   400811979   \n",
      "2   400812102  I w/S                  CHEVRON USA INC       16700   400812102   \n",
      "3   400812258      I  KERR MCGEE OIL & GAS ONSHORE LP       47120   400812258   \n",
      "4   400812277      I                 NOBLE ENERGY INC      100322   400812277   \n",
      "\n",
      "  Initial Report Date Date of Discovery  Spill Type Qtr Qtr  Section  ...  \\\n",
      "0          03/19/2015        06/04/2012  Historical  nwne         19  ...   \n",
      "1          03/19/2015        03/18/2015      Recent  SWSW         33  ...   \n",
      "2          03/19/2015        07/11/2012  Historical  nese         32  ...   \n",
      "3          03/20/2015        03/19/2015  Historical  NENW         13  ...   \n",
      "4          03/20/2015        03/18/2015      Recent  SENE         28  ...   \n",
      "\n",
      "  total_population white_population hispanic_population  \\\n",
      "0           2532.0           2144.0               295.0   \n",
      "1           5870.0           5052.0               846.0   \n",
      "2           2532.0           2144.0               295.0   \n",
      "3           1343.0           1106.0               245.0   \n",
      "4           4023.0           3581.0               612.0   \n",
      "\n",
      "   median_household_income  poverty_population unemployed_population  \\\n",
      "0                  59718.0               249.0                  86.0   \n",
      "1                  72717.0              1193.0                 253.0   \n",
      "2                  59718.0               249.0                  86.0   \n",
      "3                 140588.0                25.0                  10.0   \n",
      "4                  82865.0               268.0                  62.0   \n",
      "\n",
      "  percent_white percent_hispanic  percent_poverty unemployment_rate  \n",
      "0     84.676145        11.650869         9.834123          3.396524  \n",
      "1     86.064736        14.412266        20.323680          4.310051  \n",
      "2     84.676145        11.650869         9.834123          3.396524  \n",
      "3     82.352941        18.242740         1.861504          0.744602  \n",
      "4     89.013174        15.212528         6.661695          1.541138  \n",
      "\n",
      "[5 rows x 118 columns]\n",
      "Number of spills without matched demographic data: 0\n"
     ]
    }
   ],
   "source": [
    "\n",
    "# Create GEOID column in acs_df if not already present\n",
    "acs_df['GEOID'] = acs_df['state'] + acs_df['county'] + acs_df['tract']\n",
    "\n",
    "# Select the columns we want to keep from acs_df\n",
    "acs_columns = [\n",
    "    'GEOID', 'total_population', 'white_population', 'hispanic_population',\n",
    "    'median_household_income', 'poverty_population', 'unemployed_population',\n",
    "    'percent_white', 'percent_hispanic', 'percent_poverty', 'unemployment_rate'\n",
    "]\n",
    "\n",
    "# Merge the ACS data with the spills data\n",
    "spills_with_demographics = spills_df.merge(acs_df[acs_columns], on='GEOID', how='left')\n",
    "\n",
    "# Check the result\n",
    "print(f\"Original spills_df shape: {spills_df.shape}\")\n",
    "print(f\"spills_with_demographics shape: {spills_with_demographics.shape}\")\n",
    "print(spills_with_demographics.head())\n",
    "\n",
    "# Check for any spills that didn't get matched with demographic data\n",
    "unmatched = spills_with_demographics[spills_with_demographics['total_population'].isna()]\n",
    "print(f\"Number of spills without matched demographic data: {len(unmatched)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spills_with_demographics has been saved to the database.\n"
     ]
    }
   ],
   "source": [
    "# save spills_with_demographics to a new table in the database\n",
    "\n",
    "# Save the DataFrame to a new table in the database\n",
    "spills_with_demographics.to_sql('spills_with_demographics', engine, if_exists='replace', index=False)\n",
    "\n",
    "print(\"spills_with_demographics has been saved to the database.\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spills_with_demographics has been saved to a CSV file.\n"
     ]
    }
   ],
   "source": [
    "# create csv file\n",
    "spills_with_demographics.to_csv('spills_with_demographics.csv', index=False)\n",
    "\n",
    "print(\"spills_with_demographics has been saved to a CSV file.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gdf_spills_with_demographics has been created.\n"
     ]
    }
   ],
   "source": [
    "# create a GeoDataFrame with spills and demographics\n",
    "gdf_spills_with_demographics = gpd.GeoDataFrame(\n",
    "    spills_with_demographics,\n",
    "    geometry=gpd.points_from_xy(spills_with_demographics.Longitude, spills_with_demographics.Latitude),\n",
    "    crs=\"EPSG:4326\"\n",
    ")  \n",
    "print(\"gdf_spills_with_demographics has been created.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "# verify that the demographics data has been merged correctly\n",
    "assert not spills_with_demographics['total_population'].isna().any(), \"Some spills are missing demographic data\"\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gdf_spills_with_demographics has been verified.\n"
     ]
    }
   ],
   "source": [
    "# verify the GeoDataFrame\n",
    "assert gdf_spills_with_demographics.geometry.notnull().all(), \"Some geometries are null in gdf_spills_with_demographics\"\n",
    "assert gdf_spills_with_demographics.crs == \"EPSG:4326\", \"CRS of gdf_spills_with_demographics is not EPSG:4326\"\n",
    "print(\"gdf_spills_with_demographics has been verified.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: geoalchemy2 in /home/dadams/miniconda3/lib/python3.12/site-packages (0.18.0)\n",
      "Requirement already satisfied: SQLAlchemy>=1.4 in /home/dadams/miniconda3/lib/python3.12/site-packages (from geoalchemy2) (2.0.40)\n",
      "Requirement already satisfied: packaging in /home/dadams/miniconda3/lib/python3.12/site-packages (from geoalchemy2) (24.1)\n",
      "Requirement already satisfied: greenlet>=1 in /home/dadams/miniconda3/lib/python3.12/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (3.2.1)\n",
      "Requirement already satisfied: typing-extensions>=4.6.0 in /home/dadams/miniconda3/lib/python3.12/site-packages (from SQLAlchemy>=1.4->geoalchemy2) (4.12.2)\n"
     ]
    }
   ],
   "source": [
    "!pip install geoalchemy2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "gdf_spills_with_demographics has been saved to the database.\n",
      "gdf_spills_with_demographics has been saved to a CSV file.\n",
      "gdf_spills_with_demographics has been saved to a GeoJSON file.\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import geoalchemy2\n",
    "# Save the GeoDataFrame to a new table in the database\n",
    "gdf_spills_with_demographics.to_postgis('gdf_spills_with_demographics', engine, if_exists='replace', index=False)\n",
    "print(\"gdf_spills_with_demographics has been saved to the database.\")\n",
    "# Save the GeoDataFrame to a CSV file\n",
    "gdf_spills_with_demographics.to_csv('gdf_spills_with_demographics.csv', index=False)\n",
    "print(\"gdf_spills_with_demographics has been saved to a CSV file.\")\n",
    "# Save GeoDataFrame as a GeoJSON file\n",
    "gdf_spills_with_demographics.to_file('gdf_spills_with_demographics.geojson', driver='GeoJSON')\n",
    "print(\"gdf_spills_with_demographics has been saved to a GeoJSON file.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [],
   "source": [
    "ruca_df = pd.read_csv(\n",
    "    '/home/dadams/CSU Fullerton Dropbox/David Adams/Research Projects/colorado_spills_ejproject/github/colorado_spills/data/RUCA-codes-2020-tract.csv',  # update path\n",
    "    encoding='latin1',\n",
    "    dtype={'TractFIPS20': str}\n",
    ")\n",
    "\n",
    "# Keep and rename needed columns\n",
    "ruca_df = ruca_df.rename(columns={\n",
    "    'TractFIPS20': 'GEOID',\n",
    "    'PrimaryRUCA': 'ruca_code',\n",
    "    'PrimaryRUCADescription': 'ruca_description'\n",
    "})[['GEOID', 'ruca_code', 'ruca_description']]\n",
    "\n",
    "# Ensure GEOID is 11-character string\n",
    "ruca_df['GEOID'] = ruca_df['GEOID'].str.zfill(11)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "spills = pd.read_sql_table('gdf_spills_with_demographics', engine)\n",
    "\n",
    "# Make sure GEOID is also string-padded\n",
    "spills['GEOID'] = spills['GEOID'].astype(str).str.zfill(11)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "spills_with_ruca = spills.merge(ruca_df, on='GEOID', how='left')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def classify_rurality(code):\n",
    "    if pd.isna(code):\n",
    "        return 'Unknown'\n",
    "    code = int(code)\n",
    "    if code <= 3:\n",
    "        return 'Urban'\n",
    "    elif 4 <= code <= 6:\n",
    "        return 'Suburban'\n",
    "    else:\n",
    "        return 'Rural'\n",
    "\n",
    "spills_with_ruca['rurality'] = spills_with_ruca['ruca_code'].apply(classify_rurality)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "from shapely import wkb\n",
    "import geopandas as gpd\n",
    "\n",
    "# Convert WKBElement objects to Shapely\n",
    "spills_with_ruca['geometry'] = spills_with_ruca['geometry'].apply(lambda g: wkb.loads(bytes(g.data)) if g else None)\n",
    "\n",
    "# Now safely create GeoDataFrame\n",
    "spills_with_ruca = gpd.GeoDataFrame(spills_with_ruca, geometry='geometry', crs='EPSG:4326')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [],
   "source": [
    "spills_with_ruca.to_postgis(\n",
    "    name='spills_with_ruca',\n",
    "    con=engine,\n",
    "    if_exists='replace',\n",
    "    index=False\n",
    ")\n"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "base",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
