Capstone_3.ipynb

{
    "cells": [
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Installing relevant packages"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#install relevant packages\n! pip install geocoder",
            "execution_count": 41,
            "outputs": [
                {
                    "output_type": "stream",
                    "text": "Collecting geocoder\n\u001b[?25l  Downloading https://files.pythonhosted.org/packages/4f/6b/13166c909ad2f2d76b929a4227c952630ebaf0d729f6317eb09cbceccbab/geocoder-1.38.1-py2.py3-none-any.whl (98kB)\n\u001b[K     |\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588\u2588| 102kB 7.3MB/s ta 0:00:011\n\u001b[?25hRequirement already satisfied: requests in /opt/conda/envs/Python36/lib/python3.6/site-packages (from geocoder) (2.21.0)\nRequirement already satisfied: click in /opt/conda/envs/Python36/lib/python3.6/site-packages (from geocoder) (7.0)\nRequirement already satisfied: six in /opt/conda/envs/Python36/lib/python3.6/site-packages (from geocoder) (1.12.0)\nCollecting ratelim (from geocoder)\n  Downloading https://files.pythonhosted.org/packages/f2/98/7e6d147fd16a10a5f821db6e25f192265d6ecca3d82957a4fdd592cad49c/ratelim-0.1.6-py2.py3-none-any.whl\nRequirement already satisfied: future in /opt/conda/envs/Python36/lib/python3.6/site-packages (from geocoder) (0.17.1)\nRequirement already satisfied: urllib3<1.25,>=1.21.1 in /opt/conda/envs/Python36/lib/python3.6/site-packages (from requests->geocoder) (1.24.1)\nRequirement already satisfied: chardet<3.1.0,>=3.0.2 in /opt/conda/envs/Python36/lib/python3.6/site-packages (from requests->geocoder) (3.0.4)\nRequirement already satisfied: certifi>=2017.4.17 in /opt/conda/envs/Python36/lib/python3.6/site-packages (from requests->geocoder) (2020.6.20)\nRequirement already satisfied: idna<2.9,>=2.5 in /opt/conda/envs/Python36/lib/python3.6/site-packages (from requests->geocoder) (2.8)\nRequirement already satisfied: decorator in /opt/conda/envs/Python36/lib/python3.6/site-packages (from ratelim->geocoder) (4.3.2)\nInstalling collected packages: ratelim, geocoder\nSuccessfully installed geocoder-1.38.1 ratelim-0.1.6\n",
                    "name": "stdout"
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Imorting relevant libraries"
        },
        {
            "metadata": {
                "scrolled": true
            },
            "cell_type": "code",
            "source": "#import relevant libraries\nimport pandas as pd \nimport geocoder",
            "execution_count": 42,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Scraping webpage "
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#scraping wikipedia page using pandas\nlink = 'https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M'\ntables = pd.read_html(link)",
            "execution_count": null,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Converting to dataframe"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#converting scraped list to dataframe\ntables\ndf = pd.DataFrame(tables[0])",
            "execution_count": 58,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Removing not assigned items"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "# Assumption: all not assigned neighborhoods and boroughs occur in pair\ndf = df[df.Borough != 'Not assigned']\ndf.reset_index(inplace=True)",
            "execution_count": 59,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "df.drop(df.columns[[0]],axis=1,inplace=True)\ndf.shape",
            "execution_count": 60,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 60,
                    "data": {
                        "text/plain": "(103, 3)"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "df.head(1)",
            "execution_count": 36,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 36,
                    "data": {
                        "text/plain": "  Postal Code     Borough Neighbourhood\n0         M3A  North York     Parkwoods",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postal Code</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Geocoder not working"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#geocoder not working\nlat_lng_coords = None\nfor index, row in df.iterrows():\n    while (lat_lng_coords is None):\n        g = geocoder.google('{}, Toronto, Ontario'.format(row['Postal Code']))\n        lat_lng_coords = g.latlng\n    row['Latitude'] = lat_lng_coords[0]\n    row['Longitude'] = lat_lng_coords[1]",
            "execution_count": null,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "markdown",
            "source": "Using alternate approach"
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#using alternate approach\n!wget -q -O 'geospatial_data.csv' https://cocl.us/Geospatial_data\nlat_lng = pd.read_csv('geospatial_data.csv')",
            "execution_count": 52,
            "outputs": []
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "lat_lng.head()",
            "execution_count": 53,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 53,
                    "data": {
                        "text/plain": "  Postal Code   Latitude  Longitude\n0         M1B  43.806686 -79.194353\n1         M1C  43.784535 -79.160497\n2         M1E  43.763573 -79.188711\n3         M1G  43.770992 -79.216917\n4         M1H  43.773136 -79.239476",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postal Code</th>\n      <th>Latitude</th>\n      <th>Longitude</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M1B</td>\n      <td>43.806686</td>\n      <td>-79.194353</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M1C</td>\n      <td>43.784535</td>\n      <td>-79.160497</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M1E</td>\n      <td>43.763573</td>\n      <td>-79.188711</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M1G</td>\n      <td>43.770992</td>\n      <td>-79.216917</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M1H</td>\n      <td>43.773136</td>\n      <td>-79.239476</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "#merging dataframes\ndf = df.merge(lat_lng, on ='Postal Code', how ='left')\ndf.head()",
            "execution_count": 61,
            "outputs": [
                {
                    "output_type": "execute_result",
                    "execution_count": 61,
                    "data": {
                        "text/plain": "  Postal Code           Borough                                Neighbourhood  \\\n0         M3A        North York                                    Parkwoods   \n1         M4A        North York                             Victoria Village   \n2         M5A  Downtown Toronto                    Regent Park, Harbourfront   \n3         M6A        North York             Lawrence Manor, Lawrence Heights   \n4         M7A  Downtown Toronto  Queen's Park, Ontario Provincial Government   \n\n    Latitude  Longitude  \n0  43.753259 -79.329656  \n1  43.725882 -79.315572  \n2  43.654260 -79.360636  \n3  43.718518 -79.464763  \n4  43.662301 -79.389494  ",
                        "text/html": "<div>\n<style scoped>\n    .dataframe tbody tr th:only-of-type {\n        vertical-align: middle;\n    }\n\n    .dataframe tbody tr th {\n        vertical-align: top;\n    }\n\n    .dataframe thead th {\n        text-align: right;\n    }\n</style>\n<table border=\"1\" class=\"dataframe\">\n  <thead>\n    <tr style=\"text-align: right;\">\n      <th></th>\n      <th>Postal Code</th>\n      <th>Borough</th>\n      <th>Neighbourhood</th>\n      <th>Latitude</th>\n      <th>Longitude</th>\n    </tr>\n  </thead>\n  <tbody>\n    <tr>\n      <th>0</th>\n      <td>M3A</td>\n      <td>North York</td>\n      <td>Parkwoods</td>\n      <td>43.753259</td>\n      <td>-79.329656</td>\n    </tr>\n    <tr>\n      <th>1</th>\n      <td>M4A</td>\n      <td>North York</td>\n      <td>Victoria Village</td>\n      <td>43.725882</td>\n      <td>-79.315572</td>\n    </tr>\n    <tr>\n      <th>2</th>\n      <td>M5A</td>\n      <td>Downtown Toronto</td>\n      <td>Regent Park, Harbourfront</td>\n      <td>43.654260</td>\n      <td>-79.360636</td>\n    </tr>\n    <tr>\n      <th>3</th>\n      <td>M6A</td>\n      <td>North York</td>\n      <td>Lawrence Manor, Lawrence Heights</td>\n      <td>43.718518</td>\n      <td>-79.464763</td>\n    </tr>\n    <tr>\n      <th>4</th>\n      <td>M7A</td>\n      <td>Downtown Toronto</td>\n      <td>Queen's Park, Ontario Provincial Government</td>\n      <td>43.662301</td>\n      <td>-79.389494</td>\n    </tr>\n  </tbody>\n</table>\n</div>"
                    },
                    "metadata": {}
                }
            ]
        },
        {
            "metadata": {},
            "cell_type": "code",
            "source": "",
            "execution_count": null,
            "outputs": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "name": "python3",
            "display_name": "Python 3.6",
            "language": "python"
        },
        "language_info": {
            "name": "python",
            "version": "3.6.9",
            "mimetype": "text/x-python",
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "pygments_lexer": "ipython3",
            "nbconvert_exporter": "python",
            "file_extension": ".py"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 1
}