{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "import geopandas as gpd\n",
    "\n",
    "import os\n",
    "\n",
    "# Database connection details from zshrc environment variables\n",
    "db_name = 'colorado_spills'\n",
    "user = os.getenv('DB_USER')\n",
    "password = os.getenv('DB_PASSWORD')\n",
    "host = os.getenv('DB_HOST')\n",
    "port = os.getenv('DB_PORT')\n",
    "\n",
    "\n",
    "# Create an engine to connect to the PostgreSQL database\n",
    "engine = create_engine(f'postgresql+psycopg2://{user}:{password}@{host}:{port}/{db_name}')\n",
    "\n",
    "# Read in the spills_with_demographics as spills\n",
    "spills = pd.read_sql_table('spills_with_demographics', engine)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# use longitude and latitude to create a GeoDataFrame from spills data\n",
    "spills['geometry'] = gpd.points_from_xy(spills['Longitude'], spills['Latitude'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Create a GeoDataFrame from spills\n",
    "spills_gdf = gpd.GeoDataFrame(spills, crs='EPSG:4326') \n",
    "\n",
    "# Write the GeoDataFrame to a new table in the database\n",
    "spills_gdf.to_postgis('spills_with_demographics_geog', engine, if_exists='replace')\n",
    "\n",
    "# Close the database connection\n",
    "engine.dispose()\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Date of Discovery</th>\n",
       "      <th>Initial Report Date</th>\n",
       "      <th>Report Delay (Days)</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2014-06-11</td>\n",
       "      <td>2014-06-18</td>\n",
       "      <td>7</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2014-06-14</td>\n",
       "      <td>2014-06-19</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2014-06-14</td>\n",
       "      <td>2014-06-19</td>\n",
       "      <td>5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2014-06-19</td>\n",
       "      <td>2014-06-20</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2014-06-20</td>\n",
       "      <td>2014-06-23</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "  Date of Discovery Initial Report Date  Report Delay (Days)\n",
       "0        2014-06-11          2014-06-18                    7\n",
       "1        2014-06-14          2014-06-19                    5\n",
       "2        2014-06-14          2014-06-19                    5\n",
       "3        2014-06-19          2014-06-20                    1\n",
       "4        2014-06-20          2014-06-23                    3"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Ensure the date columns are in datetime format\n",
    "spills_gdf['Date of Discovery'] = pd.to_datetime(spills_gdf['Date of Discovery'])\n",
    "spills_gdf['Initial Report Date'] = pd.to_datetime(spills_gdf['Initial Report Date'])\n",
    "\n",
    "# Calculate the time difference in days\n",
    "spills_gdf['Report Delay (Days)'] = (spills_gdf['Initial Report Date'] - spills_gdf['Date of Discovery']).dt.days\n",
    "\n",
    "# Display the first few rows to check the new column\n",
    "spills_gdf[['Date of Discovery', 'Initial Report Date', 'Report Delay (Days)']].head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Corrected 5889 negative report delays to zero.\n"
     ]
    }
   ],
   "source": [
    "# Set negative reporting delays to zero\n",
    "spills_gdf['Report Delay (Days)'] = spills_gdf['Report Delay (Days)'].apply(lambda x: max(x, 0))\n",
    "\n",
    "# Check how many negative values were corrected\n",
    "corrected_values = (spills_gdf['Report Delay (Days)'] == 0).sum()\n",
    "print(f'Corrected {corrected_values} negative report delays to zero.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "from sklearn.neighbors import NearestNeighbors\n",
    "\n",
    "# Identify rows with negative or missing median household income\n",
    "negative_income_rows = spills_gdf[spills_gdf['median_household_income'] < 0].copy()\n",
    "\n",
    "# Identify rows with valid median household income\n",
    "valid_income_rows = spills_gdf[spills_gdf['median_household_income'] >= 0].copy()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Use latitude and longitude for nearest neighbor search\n",
    "nbrs = NearestNeighbors(n_neighbors=1, algorithm='ball_tree').fit(valid_income_rows[['Latitude', 'Longitude']])\n",
    "distances, indices = nbrs.kneighbors(negative_income_rows[['Latitude', 'Longitude']])\n",
    "\n",
    "# Extract the nearest neighbor's median household income for each negative income row\n",
    "nearest_incomes = valid_income_rows.iloc[indices.flatten()]['median_household_income'].values\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Imputed 159 negative income values using nearest neighbors.\n"
     ]
    }
   ],
   "source": [
    "# Replace the negative income values with the nearest neighbor's income\n",
    "spills_gdf.loc[negative_income_rows.index, 'median_household_income'] = nearest_incomes\n",
    "\n",
    "# Check the result\n",
    "imputed_values = len(negative_income_rows)\n",
    "print(f'Imputed {imputed_values} negative income values using nearest neighbors.')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "count    16890.000000\n",
       "mean         9.911131\n",
       "std        126.932370\n",
       "min          0.000000\n",
       "25%          0.000000\n",
       "50%          1.000000\n",
       "75%          2.000000\n",
       "max       9261.000000\n",
       "Name: Report Delay (Days), dtype: float64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Summary statistics for reporting delay\n",
    "report_delay_summary = spills_gdf['Report Delay (Days)'].describe()\n",
    "report_delay_summary\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50644 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Scatter plot of reporting delay vs. median household income\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='median_household_income', y='Report Delay (Days)', data=spills_gdf)\n",
    "plt.title('Reporting Delay vs. Median Household Income')\n",
    "plt.xlabel('Median Household Income')\n",
    "plt.ylabel('Reporting Delay (Days)')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                             OLS Regression Results                            \n",
      "===============================================================================\n",
      "Dep. Variable:     Report Delay (Days)   R-squared:                       0.000\n",
      "Model:                             OLS   Adj. R-squared:                 -0.000\n",
      "Method:                  Least Squares   F-statistic:                    0.2970\n",
      "Date:                 Fri, 09 Aug 2024   Prob (F-statistic):              0.586\n",
      "Time:                         15:02:29   Log-Likelihood:            -1.0577e+05\n",
      "No. Observations:                16890   AIC:                         2.116e+05\n",
      "Df Residuals:                    16888   BIC:                         2.116e+05\n",
      "Df Model:                            1                                         \n",
      "Covariance Type:             nonrobust                                         \n",
      "===========================================================================================\n",
      "                              coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------------------\n",
      "const                       8.1613      3.356      2.432      0.015       1.583      14.739\n",
      "median_household_income  2.207e-05   4.05e-05      0.545      0.586   -5.73e-05       0.000\n",
      "==============================================================================\n",
      "Omnibus:                    48059.926   Durbin-Watson:                   1.990\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):       3185509630.382\n",
      "Skew:                          37.898   Prob(JB):                         0.00\n",
      "Kurtosis:                    2129.202   Cond. No.                     2.85e+05\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
      "[2] The condition number is large, 2.85e+05. This might indicate that there are\n",
      "strong multicollinearity or other numerical problems.\n"
     ]
    }
   ],
   "source": [
    "import statsmodels.api as sm\n",
    "\n",
    "# Define the independent variable (adding a constant for intercept)\n",
    "X = sm.add_constant(spills_gdf['median_household_income'])\n",
    "\n",
    "# Define the dependent variable\n",
    "y = spills_gdf['Report Delay (Days)']\n",
    "\n",
    "# Fit the regression model\n",
    "model = sm.OLS(y, X).fit()\n",
    "\n",
    "# Get the summary of the model\n",
    "model_summary = model.summary()\n",
    "print(model_summary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 55796 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 53440 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 54340 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Plot spill locations by median household income\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.histplot(data=spills_gdf, x='median_household_income', bins=30, kde=True)\n",
    "plt.title('Distribution of Spills by Median Household Income')\n",
    "plt.xlabel('Median Household Income')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n",
    "\n",
    "# Plot spill locations by percent poverty\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.histplot(data=spills_gdf, x='percent_poverty', bins=30, kde=True)\n",
    "plt.title('Distribution of Spills by Percent Poverty')\n",
    "plt.xlabel('Percent Poverty')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n",
    "\n",
    "# Plot spill locations by unemployment rate\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.histplot(data=spills_gdf, x='unemployment_rate', bins=30, kde=True)\n",
    "plt.title('Distribution of Spills by Unemployment Rate')\n",
    "plt.xlabel('Unemployment Rate')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 52260 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 53684 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Plot spill locations by percent white\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.histplot(data=spills_gdf, x='percent_white', bins=30, kde=True)\n",
    "plt.title('Distribution of Spills by Percent White Population')\n",
    "plt.xlabel('Percent White')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n",
    "\n",
    "# Plot spill locations by percent hispanic\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.histplot(data=spills_gdf, x='percent_hispanic', bins=30, kde=True)\n",
    "plt.title('Distribution of Spills by Percent Hispanic Population')\n",
    "plt.xlabel('Percent Hispanic')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 50644 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 46288 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 46688 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Scatter plot of reporting delay vs. median household income\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='median_household_income', y='Report Delay (Days)', data=spills_gdf)\n",
    "plt.title('Reporting Delay vs. Median Household Income')\n",
    "plt.xlabel('Median Household Income')\n",
    "plt.ylabel('Reporting Delay (Days)')\n",
    "plt.show()\n",
    "\n",
    "# Scatter plot of reporting delay vs. percent poverty\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='percent_poverty', y='Report Delay (Days)', data=spills_gdf)\n",
    "plt.title('Reporting Delay vs. Percent Poverty')\n",
    "plt.xlabel('Percent Poverty')\n",
    "plt.ylabel('Reporting Delay (Days)')\n",
    "plt.show()\n",
    "\n",
    "# Scatter plot of reporting delay vs. unemployment rate\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='unemployment_rate', y='Report Delay (Days)', data=spills_gdf)\n",
    "plt.title('Reporting Delay vs. Unemployment Rate')\n",
    "plt.xlabel('Unemployment Rate')\n",
    "plt.ylabel('Reporting Delay (Days)')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 46016 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 48220 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Scatter plot of reporting delay vs. percent white\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='percent_white', y='Report Delay (Days)', data=spills_gdf)\n",
    "plt.title('Reporting Delay vs. Percent White Population')\n",
    "plt.xlabel('Percent White')\n",
    "plt.ylabel('Reporting Delay (Days)')\n",
    "plt.show()\n",
    "\n",
    "# Scatter plot of reporting delay vs. percent hispanic\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='percent_hispanic', y='Report Delay (Days)', data=spills_gdf)\n",
    "plt.title('Reporting Delay vs. Percent Hispanic Population')\n",
    "plt.xlabel('Percent Hispanic')\n",
    "plt.ylabel('Reporting Delay (Days)')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Select relevant columns for correlation analysis\n",
    "correlation_columns = [\n",
    "    'Report Delay (Days)', \n",
    "    'median_household_income', \n",
    "    'percent_poverty', \n",
    "    'unemployment_rate', \n",
    "    'percent_white', \n",
    "    'percent_hispanic'\n",
    "]\n",
    "\n",
    "# Create a DataFrame with the selected columns\n",
    "correlation_df = spills_gdf[correlation_columns]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Report Delay (Days)</th>\n",
       "      <th>median_household_income</th>\n",
       "      <th>percent_poverty</th>\n",
       "      <th>unemployment_rate</th>\n",
       "      <th>percent_white</th>\n",
       "      <th>percent_hispanic</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Report Delay (Days)</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.004194</td>\n",
       "      <td>-0.011415</td>\n",
       "      <td>-0.008741</td>\n",
       "      <td>-0.033242</td>\n",
       "      <td>-0.003342</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median_household_income</th>\n",
       "      <td>0.004194</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.572311</td>\n",
       "      <td>-0.218962</td>\n",
       "      <td>0.070998</td>\n",
       "      <td>-0.224321</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_poverty</th>\n",
       "      <td>-0.011415</td>\n",
       "      <td>-0.572311</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.374435</td>\n",
       "      <td>-0.093304</td>\n",
       "      <td>-0.032890</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>unemployment_rate</th>\n",
       "      <td>-0.008741</td>\n",
       "      <td>-0.218962</td>\n",
       "      <td>0.374435</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.122596</td>\n",
       "      <td>-0.131676</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_white</th>\n",
       "      <td>-0.033242</td>\n",
       "      <td>0.070998</td>\n",
       "      <td>-0.093304</td>\n",
       "      <td>0.122596</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.643538</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_hispanic</th>\n",
       "      <td>-0.003342</td>\n",
       "      <td>-0.224321</td>\n",
       "      <td>-0.032890</td>\n",
       "      <td>-0.131676</td>\n",
       "      <td>-0.643538</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Report Delay (Days)  median_household_income  \\\n",
       "Report Delay (Days)                 1.000000                 0.004194   \n",
       "median_household_income             0.004194                 1.000000   \n",
       "percent_poverty                    -0.011415                -0.572311   \n",
       "unemployment_rate                  -0.008741                -0.218962   \n",
       "percent_white                      -0.033242                 0.070998   \n",
       "percent_hispanic                   -0.003342                -0.224321   \n",
       "\n",
       "                         percent_poverty  unemployment_rate  percent_white  \\\n",
       "Report Delay (Days)            -0.011415          -0.008741      -0.033242   \n",
       "median_household_income        -0.572311          -0.218962       0.070998   \n",
       "percent_poverty                 1.000000           0.374435      -0.093304   \n",
       "unemployment_rate               0.374435           1.000000       0.122596   \n",
       "percent_white                  -0.093304           0.122596       1.000000   \n",
       "percent_hispanic               -0.032890          -0.131676      -0.643538   \n",
       "\n",
       "                         percent_hispanic  \n",
       "Report Delay (Days)             -0.003342  \n",
       "median_household_income         -0.224321  \n",
       "percent_poverty                 -0.032890  \n",
       "unemployment_rate               -0.131676  \n",
       "percent_white                   -0.643538  \n",
       "percent_hispanic                 1.000000  "
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix\n",
    "correlation_matrix = correlation_df.corr()\n",
    "\n",
    "# Display the correlation matrix\n",
    "correlation_matrix\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 99544 characters]",
      "text/plain": [
       "<Figure size 1000x800 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Set up the matplotlib figure\n",
    "plt.figure(figsize=(10, 8))\n",
    "\n",
    "# Draw the heatmap\n",
    "sns.heatmap(correlation_matrix, annot=True, fmt=\".2f\", cmap='coolwarm', vmin=-1, vmax=1)\n",
    "\n",
    "# Add title and labels\n",
    "plt.title('Correlation Matrix of Spill-Related Variables and EJ Characteristics')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Operator\n",
       "NOBLE ENERGY INC                              2402\n",
       "KERR MCGEE OIL & GAS ONSHORE LP               2243\n",
       "PDC ENERGY INC                                1182\n",
       "CAERUS PICEANCE LLC                           1051\n",
       "BONANZA CREEK ENERGY OPERATING COMPANY LLC     643\n",
       "KP KAUFFMAN COMPANY INC                        468\n",
       "CHEVRON USA INC                                467\n",
       "HIGHPOINT OPERATING CORPORATION                411\n",
       "CRESTONE PEAK RESOURCES OPERATING LLC          394\n",
       "LARAMIE ENERGY LLC                             376\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "## Count the number of spills by operator\n",
    "operator_spill_count = spills_gdf['Operator'].value_counts()\n",
    "\n",
    "# Display the top operators with the most spills\n",
    "operator_spill_count.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "county\n",
       "WELD               9850\n",
       "GARFIELD           1886\n",
       "RIO BLANCO         1034\n",
       "LAS ANIMAS          614\n",
       "ADAMS               583\n",
       "LA PLATA            496\n",
       "MESA                362\n",
       "JACKSON             271\n",
       "YUMA                270\n",
       "WASHINGTON          226\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Count the number of spills by county\n",
    "county_spill_count = spills_gdf['county'].value_counts()\n",
    "\n",
    "# Display the top counties with the most spills\n",
    "county_spill_count.head(10)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Operator\n",
       "NOBLE ENERGY INC                              2402\n",
       "KERR MCGEE OIL & GAS ONSHORE LP               2243\n",
       "PDC ENERGY INC                                1182\n",
       "CAERUS PICEANCE LLC                           1051\n",
       "BONANZA CREEK ENERGY OPERATING COMPANY LLC     643\n",
       "                                              ... \n",
       "WOLVERINE RESOURCES LLC                          2\n",
       "DAN A HUGHES COMPANY LP                          2\n",
       "SUMMIT MIDSTREAM NIOBRARA LLC                    2\n",
       "LYSTER OIL COMPANY INC                           2\n",
       "SOVEREIGN OPERATING COMPANY LLC                  2\n",
       "Name: count, Length: 275, dtype: int64"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Identify operators with more than one spill\n",
    "recurring_operator_spills = operator_spill_count[operator_spill_count > 1]\n",
    "\n",
    "# Display the operators with recurring spills\n",
    "recurring_operator_spills\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "county\n",
       "WELD               9850\n",
       "GARFIELD           1886\n",
       "RIO BLANCO         1034\n",
       "LAS ANIMAS          614\n",
       "ADAMS               583\n",
       "LA PLATA            496\n",
       "MESA                362\n",
       "JACKSON             271\n",
       "YUMA                270\n",
       "WASHINGTON          226\n",
       "MOFFAT              172\n",
       "ARAPAHOE            153\n",
       "LARIMER             122\n",
       "BOULDER             117\n",
       "CHEYENNE            106\n",
       "MONTEZUMA            97\n",
       "LOGAN                85\n",
       "LINCOLN              60\n",
       "ARCHULETA            56\n",
       "BROOMFIELD           52\n",
       "ELBERT               50\n",
       "GUNNISON             45\n",
       "KIOWA                39\n",
       "HUERFANO             32\n",
       "MORGAN               31\n",
       "BACA                 29\n",
       "PROWERS              10\n",
       "DOLORES               9\n",
       "ROUTT                 9\n",
       "KIT CARSON            8\n",
       "PITKIN                7\n",
       "DENVER                5\n",
       "FREMONT               4\n",
       "Name: count, dtype: int64"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Identify counties with more than one spill\n",
    "recurring_county_spills = county_spill_count[county_spill_count > 1]\n",
    "\n",
    "# Display the counties with recurring spills\n",
    "recurring_county_spills\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number of Spills</th>\n",
       "      <th>median_household_income</th>\n",
       "      <th>percent_poverty</th>\n",
       "      <th>percent_white</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>GEOID</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>08001008354</th>\n",
       "      <td>31</td>\n",
       "      <td>101250.0</td>\n",
       "      <td>10.757540</td>\n",
       "      <td>28.591783</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>08001008401</th>\n",
       "      <td>122</td>\n",
       "      <td>88286.0</td>\n",
       "      <td>5.918776</td>\n",
       "      <td>86.814788</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>08001008402</th>\n",
       "      <td>95</td>\n",
       "      <td>114276.0</td>\n",
       "      <td>4.121431</td>\n",
       "      <td>82.046643</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>08001008526</th>\n",
       "      <td>1</td>\n",
       "      <td>98283.0</td>\n",
       "      <td>3.801249</td>\n",
       "      <td>79.011675</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>08001008538</th>\n",
       "      <td>9</td>\n",
       "      <td>131231.0</td>\n",
       "      <td>4.150727</td>\n",
       "      <td>75.181714</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "             Number of Spills  median_household_income  percent_poverty  \\\n",
       "GEOID                                                                     \n",
       "08001008354                31                 101250.0        10.757540   \n",
       "08001008401               122                  88286.0         5.918776   \n",
       "08001008402                95                 114276.0         4.121431   \n",
       "08001008526                 1                  98283.0         3.801249   \n",
       "08001008538                 9                 131231.0         4.150727   \n",
       "\n",
       "             percent_white  \n",
       "GEOID                       \n",
       "08001008354      28.591783  \n",
       "08001008401      86.814788  \n",
       "08001008402      82.046643  \n",
       "08001008526      79.011675  \n",
       "08001008538      75.181714  "
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Group by GEOID and aggregate the number of spills and demographic characteristics\n",
    "tract_spill_summary = spills_gdf.groupby('GEOID').agg({\n",
    "    'Document #': 'count',\n",
    "    'median_household_income': 'mean',\n",
    "    'percent_poverty': 'mean',\n",
    "    'percent_white': 'mean'\n",
    "}).rename(columns={'Document #': 'Number of Spills'})\n",
    "\n",
    "# Display the summary\n",
    "tract_spill_summary.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number of Spills</th>\n",
       "      <th>median_household_income</th>\n",
       "      <th>percent_poverty</th>\n",
       "      <th>percent_white</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Number of Spills</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.157514</td>\n",
       "      <td>0.133947</td>\n",
       "      <td>-0.008777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median_household_income</th>\n",
       "      <td>-0.157514</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.625752</td>\n",
       "      <td>-0.001162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_poverty</th>\n",
       "      <td>0.133947</td>\n",
       "      <td>-0.625752</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.067320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_white</th>\n",
       "      <td>-0.008777</td>\n",
       "      <td>-0.001162</td>\n",
       "      <td>-0.067320</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Number of Spills  median_household_income  \\\n",
       "Number of Spills                 1.000000                -0.157514   \n",
       "median_household_income         -0.157514                 1.000000   \n",
       "percent_poverty                  0.133947                -0.625752   \n",
       "percent_white                   -0.008777                -0.001162   \n",
       "\n",
       "                         percent_poverty  percent_white  \n",
       "Number of Spills                0.133947      -0.008777  \n",
       "median_household_income        -0.625752      -0.001162  \n",
       "percent_poverty                 1.000000      -0.067320  \n",
       "percent_white                  -0.067320       1.000000  "
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Calculate the correlation matrix for the tract-level data\n",
    "correlation_matrix_tract = tract_spill_summary.corr()\n",
    "\n",
    "# Display the correlation matrix\n",
    "correlation_matrix_tract\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 42852 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 39476 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "image/png": "[encoded data, 42204 characters]",
      "text/plain": [
       "<Figure size 1000x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import seaborn as sns\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "# Scatter plot of Number of Spills vs. Median Household Income\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='median_household_income', y='Number of Spills', data=tract_spill_summary)\n",
    "plt.title('Number of Spills vs. Median Household Income (Census Tract Level)')\n",
    "plt.xlabel('Median Household Income')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n",
    "\n",
    "# Scatter plot of Number of Spills vs. Percent Poverty\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='percent_poverty', y='Number of Spills', data=tract_spill_summary)\n",
    "plt.title('Number of Spills vs. Percent Poverty (Census Tract Level)')\n",
    "plt.xlabel('Percent Poverty')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n",
    "\n",
    "# Scatter plot of Number of Spills vs. Percent White Population\n",
    "plt.figure(figsize=(10, 6))\n",
    "sns.scatterplot(x='percent_white', y='Number of Spills', data=tract_spill_summary)\n",
    "plt.title('Number of Spills vs. Percent White Population (Census Tract Level)')\n",
    "plt.xlabel('Percent White Population')\n",
    "plt.ylabel('Number of Spills')\n",
    "plt.show()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Number of Spills           0\n",
       "median_household_income    0\n",
       "percent_poverty            0\n",
       "percent_white              0\n",
       "dtype: int64"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Remove rows with NaN or infinite values\n",
    "tract_spill_summary_cleaned = tract_spill_summary.replace([np.inf, -np.inf], np.nan).dropna()\n",
    "\n",
    "# Check that the data is clean\n",
    "tract_spill_summary_cleaned.isna().sum()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                            OLS Regression Results                            \n",
      "==============================================================================\n",
      "Dep. Variable:       Number of Spills   R-squared:                       0.027\n",
      "Model:                            OLS   Adj. R-squared:                  0.008\n",
      "Method:                 Least Squares   F-statistic:                     1.403\n",
      "Date:                Fri, 09 Aug 2024   Prob (F-statistic):              0.244\n",
      "Time:                        15:08:59   Log-Likelihood:                -1091.6\n",
      "No. Observations:                 157   AIC:                             2191.\n",
      "Df Residuals:                     153   BIC:                             2203.\n",
      "Df Model:                           3                                         \n",
      "Covariance Type:            nonrobust                                         \n",
      "===========================================================================================\n",
      "                              coef    std err          t      P>|t|      [0.025      0.975]\n",
      "-------------------------------------------------------------------------------------------\n",
      "const                     185.0017    197.832      0.935      0.351    -205.834     575.837\n",
      "median_household_income    -0.0010      0.001     -1.178      0.241      -0.003       0.001\n",
      "percent_poverty             2.5445      4.494      0.566      0.572      -6.335      11.424\n",
      "percent_white              -0.1193      1.909     -0.063      0.950      -3.890       3.652\n",
      "==============================================================================\n",
      "Omnibus:                      196.289   Durbin-Watson:                   1.335\n",
      "Prob(Omnibus):                  0.000   Jarque-Bera (JB):             6950.948\n",
      "Skew:                           4.985   Prob(JB):                         0.00\n",
      "Kurtosis:                      34.035   Cond. No.                     9.33e+05\n",
      "==============================================================================\n",
      "\n",
      "Notes:\n",
      "[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.\n",
      "[2] The condition number is large, 9.33e+05. This might indicate that there are\n",
      "strong multicollinearity or other numerical problems.\n"
     ]
    }
   ],
   "source": [
    "# Define the independent variables (with a constant for intercept)\n",
    "X = tract_spill_summary_cleaned[['median_household_income', 'percent_poverty', 'percent_white']]\n",
    "X = sm.add_constant(X)\n",
    "\n",
    "# Define the dependent variable\n",
    "y = tract_spill_summary_cleaned['Number of Spills']\n",
    "\n",
    "# Fit the regression model\n",
    "model = sm.OLS(y, X).fit()\n",
    "\n",
    "# Get the summary of the model\n",
    "model_summary = model.summary()\n",
    "print(model_summary)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Number of Spills</th>\n",
       "      <th>median_household_income</th>\n",
       "      <th>percent_poverty</th>\n",
       "      <th>percent_white</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Number of Spills</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.157012</td>\n",
       "      <td>0.133947</td>\n",
       "      <td>-0.008777</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>median_household_income</th>\n",
       "      <td>-0.157012</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.625752</td>\n",
       "      <td>-0.001162</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_poverty</th>\n",
       "      <td>0.133947</td>\n",
       "      <td>-0.625752</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>-0.067320</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>percent_white</th>\n",
       "      <td>-0.008777</td>\n",
       "      <td>-0.001162</td>\n",
       "      <td>-0.067320</td>\n",
       "      <td>1.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         Number of Spills  median_household_income  \\\n",
       "Number of Spills                 1.000000                -0.157012   \n",
       "median_household_income         -0.157012                 1.000000   \n",
       "percent_poverty                  0.133947                -0.625752   \n",
       "percent_white                   -0.008777                -0.001162   \n",
       "\n",
       "                         percent_poverty  percent_white  \n",
       "Number of Spills                0.133947      -0.008777  \n",
       "median_household_income        -0.625752      -0.001162  \n",
       "percent_poverty                 1.000000      -0.067320  \n",
       "percent_white                  -0.067320       1.000000  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Re-calculate the correlation matrix\n",
    "correlation_matrix_cleaned = tract_spill_summary_cleaned.corr()\n",
    "\n",
    "# Display the cleaned correlation matrix\n",
    "correlation_matrix_cleaned\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "funkyfunk",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
