{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Document #</th>\n",
       "      <th>Report</th>\n",
       "      <th>Operator</th>\n",
       "      <th>Operator #</th>\n",
       "      <th>Tracking #</th>\n",
       "      <th>Initial Report Date</th>\n",
       "      <th>Date of Discovery</th>\n",
       "      <th>Spill Type</th>\n",
       "      <th>Qtr Qtr</th>\n",
       "      <th>Section</th>\n",
       "      <th>...</th>\n",
       "      <th>Soil Excavated</th>\n",
       "      <th>Offsite Disposal</th>\n",
       "      <th>Onsite Treatment</th>\n",
       "      <th>Other Disposition</th>\n",
       "      <th>Other Disposition Description</th>\n",
       "      <th>Ground Water Removed</th>\n",
       "      <th>Surface Water Removed</th>\n",
       "      <th>Corrective Actions Completed</th>\n",
       "      <th>Approved Form 27</th>\n",
       "      <th>Form 27 Project Number</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>400628833</td>\n",
       "      <td>I w/S</td>\n",
       "      <td>GREAT WESTERN OPERATING COMPANY LLC</td>\n",
       "      <td>10110</td>\n",
       "      <td>400628833</td>\n",
       "      <td>06/18/2014</td>\n",
       "      <td>06/11/2014</td>\n",
       "      <td>Historical</td>\n",
       "      <td>SESE</td>\n",
       "      <td>2</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>No work performed yet</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2141471.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>400629910</td>\n",
       "      <td>I</td>\n",
       "      <td>COLORADO OIL &amp; GAS CONSERVATION COMMISSION</td>\n",
       "      <td>5</td>\n",
       "      <td>400629910</td>\n",
       "      <td>06/19/2014</td>\n",
       "      <td>06/14/2014</td>\n",
       "      <td>Historical</td>\n",
       "      <td>NWNE</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>400629998</td>\n",
       "      <td>S</td>\n",
       "      <td>COLORADO OIL &amp; GAS CONSERVATION COMMISSION</td>\n",
       "      <td>5</td>\n",
       "      <td>400629910</td>\n",
       "      <td>06/19/2014</td>\n",
       "      <td>06/14/2014</td>\n",
       "      <td>Historical</td>\n",
       "      <td>NWNE</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>200.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>None</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>True</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>400630821</td>\n",
       "      <td>I</td>\n",
       "      <td>KERR MCGEE GATHERING LLC</td>\n",
       "      <td>47121</td>\n",
       "      <td>400630821</td>\n",
       "      <td>06/20/2014</td>\n",
       "      <td>06/19/2014</td>\n",
       "      <td>Recent</td>\n",
       "      <td>NWNW</td>\n",
       "      <td>15</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>400631216</td>\n",
       "      <td>I</td>\n",
       "      <td>NOBLE ENERGY INC</td>\n",
       "      <td>100322</td>\n",
       "      <td>400631216</td>\n",
       "      <td>06/23/2014</td>\n",
       "      <td>06/20/2014</td>\n",
       "      <td>Recent</td>\n",
       "      <td>NENW</td>\n",
       "      <td>36</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>None</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>False</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2141511.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 106 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Document # Report                                    Operator  Operator #  \\\n",
       "0   400628833  I w/S         GREAT WESTERN OPERATING COMPANY LLC       10110   \n",
       "1   400629910      I  COLORADO OIL & GAS CONSERVATION COMMISSION           5   \n",
       "2   400629998      S  COLORADO OIL & GAS CONSERVATION COMMISSION           5   \n",
       "3   400630821      I                    KERR MCGEE GATHERING LLC       47121   \n",
       "4   400631216      I                            NOBLE ENERGY INC      100322   \n",
       "\n",
       "   Tracking # Initial Report Date Date of Discovery  Spill Type Qtr Qtr  \\\n",
       "0   400628833          06/18/2014        06/11/2014  Historical  SESE     \n",
       "1   400629910          06/19/2014        06/14/2014  Historical  NWNE     \n",
       "2   400629910          06/19/2014        06/14/2014  Historical  NWNE     \n",
       "3   400630821          06/20/2014        06/19/2014      Recent  NWNW     \n",
       "4   400631216          06/23/2014        06/20/2014      Recent  NENW     \n",
       "\n",
       "   Section  ... Soil Excavated Offsite Disposal Onsite Treatment  \\\n",
       "0        2  ...            NaN              0.0              0.0   \n",
       "1        1  ...            NaN              NaN              NaN   \n",
       "2        1  ...          200.0              1.0              0.0   \n",
       "3       15  ...            NaN              NaN              NaN   \n",
       "4       36  ...            NaN              NaN              NaN   \n",
       "\n",
       "   Other Disposition  Other Disposition Description Ground Water Removed  \\\n",
       "0                1.0          No work performed yet                  NaN   \n",
       "1                NaN                           None                  NaN   \n",
       "2                0.0                           None                  0.0   \n",
       "3                NaN                           None                  NaN   \n",
       "4                NaN                           None                  NaN   \n",
       "\n",
       "  Surface Water Removed Corrective Actions Completed  Approved Form 27  \\\n",
       "0                   NaN                        False               NaN   \n",
       "1                   NaN                        False               NaN   \n",
       "2                   0.0                         True               NaN   \n",
       "3                   NaN                        False               NaN   \n",
       "4                   NaN                        False               NaN   \n",
       "\n",
       "  Form 27 Project Number  \n",
       "0              2141471.0  \n",
       "1                    NaN  \n",
       "2                    NaN  \n",
       "3                    NaN  \n",
       "4              2141511.0  \n",
       "\n",
       "[5 rows x 106 columns]"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "from sqlalchemy import create_engine\n",
    "import geopandas as gpd\n",
    "\n",
    "import os\n",
    "\n",
    "# Database connection details from zshrc environment variables\n",
    "db_name = 'colorado_spills'\n",
    "user = os.getenv('DB_USER')\n",
    "password = os.getenv('DB_PASSWORD')\n",
    "host = os.getenv('DB_HOST')\n",
    "port = os.getenv('DB_PORT')\n",
    "\n",
    "\n",
    "# Create an engine to connect to the PostgreSQL database\n",
    "engine = create_engine(f'postgresql+psycopg2://{user}:{password}@{host}:{port}/{db_name}')\n",
    "\n",
    "# Function to load data from a table\n",
    "def load_table(table_name):\n",
    "    query = f'SELECT * FROM {table_name}'\n",
    "    df = pd.read_sql(query, engine)\n",
    "    return df\n",
    "\n",
    "# Load the spills data\n",
    "spills_df = load_table('spills')\n",
    "spills_df.head()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['Document #', 'Report', 'Operator', 'Operator #', 'Tracking #', 'Initial Report Date', 'Date of Discovery', 'Spill Type', 'Qtr Qtr', 'Section', 'Township', 'range', 'meridian', 'Latitude', 'Longitude', 'Municipality', 'county', 'Facility Type', 'Facility ID', 'API County Code', 'API Sequence Number', 'Spilled outside of berms', 'More than five barrels spilled', 'Oil Spill Volume', 'Condensate Spill Volume', 'Flow Back Spill Volume', 'Produced Water Spill Volume', 'E&P Waste Spill Volume', 'Other Waste', 'Drilling Fluid Spill Volume', 'Current Land Use', 'Other Land Use', 'Weather Conditions', 'Surface Owner', 'Surface Owner Other', 'Waters of the State', 'Residence / Occupied Structure', 'livestock', 'Public Byway', 'Surface Water Supply Area', 'Spill Description', 'Supplemental Report Date', 'Oil BBLs Spilled', 'Oil BBLs Recovered', 'Oil Unknown', 'Condensate BBLs Spilled', 'Condensate BBLs Recovered', 'Condensate Unknown', 'Produced Water BBLs Spilled', 'Produced Water BBLs Recovered', 'Produced Water Unknown', 'Drilling Fluid BBLs Spilled', 'Drilling Fluid BBLs Recovered', 'Drilling Fluid Unknown', 'Flow Back Fluid BBLs Spilled', 'Flow Back Fluid BBLs Recovered', 'Flow Back Fluid Unkown', 'Other E&P Waste BBLS Spilled', 'Other E&P Waste BBLS Recovered', 'Other E&P Waste Unknown', 'Other E&P Waste', 'Spill Contained within Berm', 'Emergency Pit Constructed', 'soil', 'groundwater', 'Surface Water', 'Dry Drainage Feature', 'Surface Area Length', 'Surface Area Width', 'Depth of Impact in Feet', 'Depth of Impact in Inches', 'Area Depth Determined', 'Geology Description', 'Depth to Groundwater', 'Water wells in area', 'Water Wells', 'Water Wells None', 'Surface Water Near', 'Surface Water None', 'Wetlands', 'Wetlands None', 'Springs', 'Springs None', 'Livestock Near', 'Livestock None', 'Occupied Buildings', 'Occupied Buildings None', 'Additional Spill Details', 'Supplemental Report Date CA', 'Human Error', 'Equipment Failure', 'Historical Unkown', 'Other', 'Other Description', 'Root Cause', 'Preventative Measures', 'Soil Excavated', 'Offsite Disposal', 'Onsite Treatment', 'Other Disposition', 'Other Disposition Description', 'Ground Water Removed', 'Surface Water Removed', 'Corrective Actions Completed', 'Approved Form 27', 'Form 27 Project Number']\n"
     ]
    }
   ],
   "source": [
    "# Assuming you have a DataFrame named df\n",
    "print(spills_df.columns.tolist())\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Census tract data has been added to spills_df.\n",
      "A plot of spills per tract has been generated.\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 223136 characters]",
      "text/plain": [
       "<Figure size 1500x1000 with 2 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "import geopandas as gpd\n",
    "import pandas as pd\n",
    "import us\n",
    "import requests\n",
    "from io import BytesIO\n",
    "from zipfile import ZipFile\n",
    "import os\n",
    "import shutil\n",
    "\n",
    "# Convert spills_df to a GeoDataFrame\n",
    "gdf = gpd.GeoDataFrame(\n",
    "    spills_df, \n",
    "    geometry=gpd.points_from_xy(spills_df.Longitude, spills_df.Latitude),\n",
    "    crs=\"EPSG:4326\"\n",
    ")\n",
    "\n",
    "# Get Colorado's FIPS code\n",
    "colorado_fips = us.states.CO.fips\n",
    "\n",
    "# Create a temporary directory for the shapefiles\n",
    "temp_dir = \"colorado_tracts_temp\"\n",
    "if not os.path.exists(temp_dir):\n",
    "    os.makedirs(temp_dir)\n",
    "\n",
    "try:\n",
    "    # Download Colorado tract shapefile\n",
    "    url = f\"https://www2.census.gov/geo/tiger/TIGER2021/TRACT/tl_2021_{colorado_fips}_tract.zip\"\n",
    "    r = requests.get(url)\n",
    "    z = ZipFile(BytesIO(r.content))\n",
    "    z.extractall(temp_dir)\n",
    "\n",
    "    # Find the .shp file in the extracted contents\n",
    "    shp_file = next(f for f in os.listdir(temp_dir) if f.endswith('.shp'))\n",
    "    \n",
    "    # Read the shapefile\n",
    "    tracts = gpd.read_file(os.path.join(temp_dir, shp_file))\n",
    "    tracts = tracts.to_crs(gdf.crs)\n",
    "\n",
    "    # Perform spatial join\n",
    "    result = gpd.sjoin(gdf, tracts, how=\"left\", predicate=\"within\")\n",
    "\n",
    "    # Add census tract information to the original DataFrame\n",
    "    spills_df['GEOID'] = result['GEOID']\n",
    "    spills_df['TRACT_NAME'] = result['NAME']\n",
    "\n",
    "    # Count spills per census tract\n",
    "    spills_per_tract = spills_df.groupby('GEOID').size().reset_index(name='spill_count')\n",
    "    tracts_with_spills = tracts.merge(spills_per_tract, on='GEOID', how='left')\n",
    "    tracts_with_spills['spill_count'] = tracts_with_spills['spill_count'].fillna(0)\n",
    "\n",
    "    # Basic plot\n",
    "    tracts_with_spills.plot(column='spill_count', legend=True, figsize=(15, 10))\n",
    "\n",
    "finally:\n",
    "    # Clean up: remove the temporary directory\n",
    "    shutil.rmtree(temp_dir, ignore_errors=True)\n",
    "\n",
    "print(\"Census tract data has been added to spills_df.\")\n",
    "print(\"A plot of spills per tract has been generated.\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "         GEOID  total_population  percent_white  percent_hispanic  \\\n",
      "0  08001008354            4211.0      28.591783         23.557350   \n",
      "1  08001008401            5491.0      86.814788         14.059370   \n",
      "2  08001008402            4974.0      82.046643         14.053076   \n",
      "3  08001008526            7366.0      79.011675         16.806951   \n",
      "4  08001008538           10456.0      75.181714         30.174063   \n",
      "\n",
      "   median_household_income  percent_poverty  unemployment_rate  spill_count  \n",
      "0                 101250.0        10.757540           0.379957           31  \n",
      "1                  88286.0         5.918776           1.456930          122  \n",
      "2                 114276.0         4.121431           2.211500           95  \n",
      "3                  98283.0         3.801249           2.104263            1  \n",
      "4                 131231.0         4.150727           3.471691            9  \n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import geopandas as gpd\n",
    "import requests\n",
    "from census import Census\n",
    "from us import states\n",
    "\n",
    "import os\n",
    "\n",
    "# census API key from zshrc environment variable\n",
    "\n",
    "c = Census(os.getenv('CENSUS_API_KEY'))\n",
    "\n",
    "# Get unique GEOIDs from your spills data\n",
    "unique_geoids = spills_df['GEOID'].unique()\n",
    "\n",
    "# Function to get state and county FIPS codes from GEOID\n",
    "def get_state_county(geoid):\n",
    "    return geoid[:2], geoid[2:5]\n",
    "\n",
    "# Prepare list of unique state and county codes\n",
    "state_counties = list(set([get_state_county(geoid) for geoid in unique_geoids]))\n",
    "\n",
    "# List of variables to fetch (you can modify this based on your specific needs)\n",
    "variables = [\n",
    "    'B01003_001E',  # Total Population\n",
    "    'B02001_002E',  # White Population\n",
    "    'B03002_012E',  # Hispanic or Latino Population\n",
    "    'B19013_001E',  # Median Household Income\n",
    "    'B17001_002E',  # Population for whom poverty status is determined\n",
    "    'B23025_005E',  # Unemployed Population\n",
    "]\n",
    "\n",
    "# Fetch ACS data\n",
    "acs_data = []\n",
    "for state, county in state_counties:\n",
    "    acs_data.extend(c.acs5.state_county_tract(\n",
    "        variables,\n",
    "        state,\n",
    "        county,\n",
    "        Census.ALL,\n",
    "        year=2021\n",
    "    ))\n",
    "\n",
    "# Convert to DataFrame\n",
    "acs_df = pd.DataFrame(acs_data)\n",
    "\n",
    "# Rename columns\n",
    "acs_df = acs_df.rename(columns={\n",
    "    'B01003_001E': 'total_population',\n",
    "    'B02001_002E': 'white_population',\n",
    "    'B03002_012E': 'hispanic_population',\n",
    "    'B19013_001E': 'median_household_income',\n",
    "    'B17001_002E': 'poverty_population',\n",
    "    'B23025_005E': 'unemployed_population'\n",
    "})\n",
    "\n",
    "# Calculate percentages and rates\n",
    "acs_df['percent_white'] = acs_df['white_population'] / acs_df['total_population'] * 100\n",
    "acs_df['percent_hispanic'] = acs_df['hispanic_population'] / acs_df['total_population'] * 100\n",
    "acs_df['percent_poverty'] = acs_df['poverty_population'] / acs_df['total_population'] * 100\n",
    "acs_df['unemployment_rate'] = acs_df['unemployed_population'] / acs_df['total_population'] * 100\n",
    "\n",
    "# Create GEOID column\n",
    "acs_df['GEOID'] = acs_df['state'] + acs_df['county'] + acs_df['tract']\n",
    "\n",
    "# Merge ACS data with spills data\n",
    "spills_ej_df = spills_df.merge(acs_df, on='GEOID', how='left')\n",
    "\n",
    "# Aggregate spills data by census tract\n",
    "spills_by_tract = spills_ej_df.groupby('GEOID').agg({\n",
    "    'total_population': 'first',\n",
    "    'percent_white': 'first',\n",
    "    'percent_hispanic': 'first',\n",
    "    'median_household_income': 'first',\n",
    "    'percent_poverty': 'first',\n",
    "    'unemployment_rate': 'first',\n",
    "    'Document #': 'count'  # Assuming this is a unique identifier for each spill\n",
    "}).reset_index()\n",
    "\n",
    "spills_by_tract = spills_by_tract.rename(columns={'Document #': 'spill_count'})\n",
    "\n",
    "print(spills_by_tract.head())\n",
    "\n",
    "# You can now use spills_by_tract for further analysis and visualization"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(158, 8)"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "spills_by_tract.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Original spills_df shape: (16890, 108)\n",
      "spills_with_demographics shape: (16890, 118)\n",
      "   Document # Report                                    Operator  Operator #  \\\n",
      "0   400628833  I w/S         GREAT WESTERN OPERATING COMPANY LLC       10110   \n",
      "1   400629910      I  COLORADO OIL & GAS CONSERVATION COMMISSION           5   \n",
      "2   400629998      S  COLORADO OIL & GAS CONSERVATION COMMISSION           5   \n",
      "3   400630821      I                    KERR MCGEE GATHERING LLC       47121   \n",
      "4   400631216      I                            NOBLE ENERGY INC      100322   \n",
      "\n",
      "   Tracking # Initial Report Date Date of Discovery  Spill Type Qtr Qtr  \\\n",
      "0   400628833          06/18/2014        06/11/2014  Historical  SESE     \n",
      "1   400629910          06/19/2014        06/14/2014  Historical  NWNE     \n",
      "2   400629910          06/19/2014        06/14/2014  Historical  NWNE     \n",
      "3   400630821          06/20/2014        06/19/2014      Recent  NWNW     \n",
      "4   400631216          06/23/2014        06/20/2014      Recent  NENW     \n",
      "\n",
      "   Section  ... total_population white_population hispanic_population  \\\n",
      "0        2  ...           5491.0           4767.0               772.0   \n",
      "1        1  ...           7335.0           6302.0              2011.0   \n",
      "2        1  ...           7335.0           6302.0              2011.0   \n",
      "3       15  ...            427.0            222.0               183.0   \n",
      "4       36  ...           3470.0           3105.0               167.0   \n",
      "\n",
      "   median_household_income  poverty_population unemployed_population  \\\n",
      "0                  88286.0               325.0                  80.0   \n",
      "1                  71440.0               831.0                 166.0   \n",
      "2                  71440.0               831.0                 166.0   \n",
      "3             -666666666.0                 0.0                   0.0   \n",
      "4                 122604.0               129.0                  13.0   \n",
      "\n",
      "  percent_white percent_hispanic  percent_poverty unemployment_rate  \n",
      "0     86.814788        14.059370         5.918776          1.456930  \n",
      "1     85.916837        27.416496        11.329243          2.263122  \n",
      "2     85.916837        27.416496        11.329243          2.263122  \n",
      "3     51.990632        42.857143         0.000000          0.000000  \n",
      "4     89.481268         4.812680         3.717579          0.374640  \n",
      "\n",
      "[5 rows x 118 columns]\n",
      "Number of spills without matched demographic data: 0\n"
     ]
    }
   ],
   "source": [
    "# Assuming you have already fetched the ACS data as described earlier\n",
    "# and it's stored in a DataFrame called acs_df\n",
    "\n",
    "# Create GEOID column in acs_df if not already present\n",
    "acs_df['GEOID'] = acs_df['state'] + acs_df['county'] + acs_df['tract']\n",
    "\n",
    "# Select the columns we want to keep from acs_df\n",
    "acs_columns = [\n",
    "    'GEOID', 'total_population', 'white_population', 'hispanic_population',\n",
    "    'median_household_income', 'poverty_population', 'unemployed_population',\n",
    "    'percent_white', 'percent_hispanic', 'percent_poverty', 'unemployment_rate'\n",
    "]\n",
    "\n",
    "# Merge the ACS data with the spills data\n",
    "spills_with_demographics = spills_df.merge(acs_df[acs_columns], on='GEOID', how='left')\n",
    "\n",
    "# Check the result\n",
    "print(f\"Original spills_df shape: {spills_df.shape}\")\n",
    "print(f\"spills_with_demographics shape: {spills_with_demographics.shape}\")\n",
    "print(spills_with_demographics.head())\n",
    "\n",
    "# Check for any spills that didn't get matched with demographic data\n",
    "unmatched = spills_with_demographics[spills_with_demographics['total_population'].isna()]\n",
    "print(f\"Number of spills without matched demographic data: {len(unmatched)}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spills_with_demographics has been saved to the database.\n"
     ]
    }
   ],
   "source": [
    "# save spills_with_demographics to a new table in the database\n",
    "\n",
    "# Save the DataFrame to a new table in the database\n",
    "spills_with_demographics.to_sql('spills_with_demographics', engine, if_exists='replace', index=False)\n",
    "\n",
    "print(\"spills_with_demographics has been saved to the database.\")\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "spills_with_demographics has been saved to a CSV file.\n"
     ]
    }
   ],
   "source": [
    "# create csv file\n",
    "spills_with_demographics.to_csv('spills_with_demographics.csv', index=False)\n",
    "\n",
    "print(\"spills_with_demographics has been saved to a CSV file.\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "funkyfunk",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
