project1_functions.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from geopy.geocoders import Nominatim\n",
    "import datetime"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_data(week_nums):\n",
    "    url = \"http://web.mta.info/developers/data/nyct/turnstile/turnstile_{}.txt\"\n",
    "    dfs = []\n",
    "    for week_num in week_nums:\n",
    "        file_url = url.format(week_num)\n",
    "        dfs.append(pd.read_csv(file_url))\n",
    "    return dfs\n",
    "\n",
    "def clean_dfs(dfs):\n",
    "    from clean_data import clean_data\n",
    "    for df in dfs:\n",
    "        clean_data(df)\n",
    "    return dfs\n",
    "\n",
    "def concat_dfs(dfs):\n",
    "    import pandas as pd\n",
    "    df_sorted = pd.concat(dfs)\n",
    "    df_sorted['weekly_turns']= df_sorted['total_turns']/len(df_list)\n",
    "    return df_sorted\n",
    "\n",
    "def get_boroughs(df_sorted):\n",
    "    '''\n",
    "    Description: Function that adds a \"borough\" column \\\n",
    "                    to cleaned df from latitude/longitude column\n",
    "    Args: df_sorted - sorted dataframe with 'location_string' column\n",
    "    Returns: df_sorted - returns the input df with additional \"borough\" column\n",
    "    '''\n",
    "    from geopy.geocoders import Nominatim\n",
    "    geolocator = Nominatim(user_agent=\"project1\")\n",
    "    df_sorted['location_string_copy'] = df_sorted['location_string'].copy()\n",
    "    loc = df_sorted.location_string.dropna().unique()\n",
    "    loc\n",
    "    borough_set = set(['Manhattan', 'Brooklyn', 'Queens', 'The Bronx', 'Staten Island'])\n",
    "    for i, l in enumerate(loc):\n",
    "        sub = str(geolocator.reverse(l))\n",
    "        sub_split = sub.split(', ')\n",
    "        station_borough = list(borough_set.intersection(sub_split))\n",
    "        if len(station_borough) > 1:\n",
    "            station_borough = station_borough[0]\n",
    "        if station_borough == []:\n",
    "            station_borough = 'Unknown'\n",
    "        df_sorted.location_string_copy.replace({l:station_borough}, inplace=True)\n",
    "    df_sorted = df_sorted.rename(columns={'location_string_copy':'borough'})\n",
    "    return df_sorted\n",
    "        \n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_total_traffic_by_station(df_sorted, metric='total_turns', borough=False):\n",
    "    '''\n",
    "    Function to plot the total turnstile turns for all stations, or a subset of stations belonging to \n",
    "        the same borough\n",
    "    Args: df_sorted - cleaned df to be plotted\n",
    "            metric - 'y value to be plotted (eg. total turns, weekly turns, entry/exit ratio)\n",
    "            borough - indicate borough to be plotted (e.g. 'Manhattan') defaults to false, which is all stations\n",
    "    \n",
    "    '''\n",
    "    import numpy as np\n",
    "    import matplotlib.pyplot as plt\n",
    "    %matplotlib inline\n",
    "    \n",
    "    if borough:\n",
    "        df_sorted = df_sorted[df_sorted['borough']==borough]\n",
    "    \n",
    "    #Group by Station-Line Name Combo... eg, 34th Penn Station is one huge station but the street team\n",
    "    #cannot be at the A-C-E entrance and the 1-2-3 entrance simultaneously, so that traffic should be \n",
    "    #split accordingly\n",
    "    df_sorted_station = df_sorted.groupby(['station_unique', 'turnstiles']).agg({'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                  'total_turns':'sum', 'weekly_turns':'sum'})\n",
    "    #Reset turnstiles index and aggregate again to count number of turnstiles per station\n",
    "    #Add turns per turnstile column\n",
    "    df_sorted_station = df_sorted_station.reset_index(level='turnstiles').groupby('station_unique').agg({'turnstiles':'count', 'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                  'total_turns':'sum', 'weekly_turns':'sum'})\n",
    "    df_sorted_station['weekly_turns_per_turnstile'] = df_sorted_station['weekly_turns']/df_sorted_station['turnstiles']\n",
    "    df_sorted_station['total_turns_per_turnstile'] = df_sorted_station['total_turns']/df_sorted_station['turnstiles']\n",
    "    df_sorted_station['entry_ratio'] = df_sorted_station['entries_diff']/df_sorted_station['total_turns']\n",
    "    \n",
    "    #Set y values to be total entries + exits,  x values to be # of turnstiles at each station\n",
    "    #names will be the station name labels for each scatter plot point\n",
    "    x = df_sorted_station.turnstiles\n",
    "    y = df_sorted_station[metric]\n",
    "    names = df_sorted_station.index\n",
    "\n",
    "    #initiate scatter plot\n",
    "    %matplotlib notebook\n",
    "    fig,ax = plt.subplots(figsize=(10,8))\n",
    "    sc = ax.scatter(x,y,s=100, alpha=0.3)\n",
    "\n",
    "    \n",
    "    \n",
    "    #Borrowed code from https://stackoverflow.com/questions/7908636/possible-to \\\n",
    "    #-make-labels-appear-when-hovering-over-a-point-in-matplotlib\n",
    "    annot = ax.annotate(\"\", xy=(0,0), xytext=(-30,-30), size = 15, textcoords=\"offset points\", \\\n",
    "                    bbox=dict(boxstyle=\"round\", fc=\"w\"), arrowprops=dict(arrowstyle=\"->\"))\n",
    "    annot.set_visible(False)\n",
    "\n",
    "    def update_annot(ind):\n",
    "        pos = sc.get_offsets()[ind[\"ind\"][0]]\n",
    "        annot.xy = pos\n",
    "        text = \"{}\".format(\" \".join([names[n] for n in ind[\"ind\"]]))\n",
    "        annot.set_text(text)\n",
    "        annot.get_bbox_patch().set_alpha(0.4)\n",
    "    \n",
    "    def hover(event):\n",
    "        vis = annot.get_visible()\n",
    "        if event.inaxes == ax:\n",
    "            cont, ind = sc.contains(event)\n",
    "            if cont:\n",
    "                update_annot(ind)\n",
    "                annot.set_visible(True)\n",
    "                fig.canvas.draw_idle()\n",
    "            else:\n",
    "                if vis:\n",
    "                    annot.set_visible(False)\n",
    "                    fig.canvas.draw_idle()\n",
    "                \n",
    "    fig.canvas.mpl_connect(\"motion_notify_event\", hover)\n",
    "\n",
    "    ax.set_xlabel('Number of turnstiles')\n",
    "    ax.set_ylabel('Total Weekly Turnstile Turns')\n",
    "    ax.grid(True)\n",
    "    plt.show()\n",
    "    \n",
    "# plot_total_traffic_by_station(df_sorted, metric='weekly_turns', borough='Brooklyn')\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_traffic_density(df_sorted, stations = 10, metric='weekly_turns_per_turnstile', borough=False):\n",
    "    import numpy as np\n",
    "    import matplotlib.pyplot as plt\n",
    "    %matplotlib inline\n",
    "    \n",
    "    if borough:\n",
    "        df_sorted = df_sorted[df_sorted['borough']==borough] \n",
    "    \n",
    "    \n",
    "    #Group by Station-Line Name Combo... eg, 34th Penn Station is one huge station but the street team\n",
    "    #cannot be at the A-C-E entrance and the 1-2-3 entrance simultaneously, so that traffic should be \n",
    "    #split accordingly\n",
    "    df_sorted_station = df_sorted.groupby(['station_unique', 'turnstiles']).agg({'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                  'total_turns':'sum', 'weekly_turns':'sum'})\n",
    "    #Reset turnstiles index and aggregate again to count number of turnstiles per station\n",
    "    #Add turns per turnstile column\n",
    "    df_sorted_station = df_sorted_station.reset_index(level='turnstiles').groupby('station_unique').agg({'turnstiles':'count', 'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                  'total_turns':'sum', 'weekly_turns':'sum'})\n",
    "    df_sorted_station['weekly_turns_per_turnstile'] = df_sorted_station['weekly_turns']/df_sorted_station['turnstiles']\n",
    "    df_sorted_station['total_turns_per_turnstile'] = df_sorted_station['total_turns']/df_sorted_station['turnstiles']\n",
    "    df_sorted_station['entry_ratio'] = df_sorted_station['entries_diff']/df_sorted_station['total_turns']\n",
    "    \n",
    "    #Set y values to be total entries + exits,  x values to be # of turnstiles at each station\n",
    "    #names will be the station name labels for each scatter plot point\n",
    "    x = df_sorted_station.turnstiles\n",
    "    y = df_sorted_station[metric]\n",
    "    \n",
    "    \n",
    "\n",
    "\n",
    "#Plot bar graph of the traffic density, or turns per turnstile, for busiest subway stations\n",
    "    df_busiest = df_sorted_station.sort_values('total_turns', ascending=False).head(stations)\n",
    "    fig,ax = plt.subplots(figsize=(15,stations/2))\n",
    "    y_pos = np.arange(len(df_busiest.total_turns_per_turnstile))\n",
    "    ax.barh(y_pos, df_busiest.total_turns_per_turnstile)\n",
    "    ax.set_yticks(y_pos)\n",
    "    ax.set_yticklabels(df_busiest.index)\n",
    "    ax.set_xlabel('Turns Per Turnstile')\n",
    "    ax.invert_yaxis() #set labels so that busiest stations are at top\n",
    "\n",
    "# plot_traffic_density(df_sorted, stations = 40, borough = 'Brooklyn')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_traffic_by_day_of_week(df_sorted, num_stations=10, custom_stations=None, borough=False, title = 'Title', \\\n",
    "                                ylabel = 'Total Turnstile Turns'):\n",
    "    import matplotlib.pyplot as plt\n",
    "    %matplotlib inline\n",
    "    \n",
    "    if borough:\n",
    "        df_sorted = df_sorted[df_sorted['borough']==borough]\n",
    "        \n",
    "    df_sorted_station = df_sorted.groupby(['station_unique', 'turnstiles', 'day_of_week'\\\n",
    "                                              ]).agg({'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                                  'total_turns':'sum'})\n",
    "\n",
    "    df_sorted_station = df_sorted_station.reset_index(level='turnstiles').groupby(['station_unique', \\\n",
    "                                                    'day_of_week']).agg({'entries_diff': \\\n",
    "                                                        'sum', 'exits_diff':'sum',\\\n",
    "                                                          'total_turns':'sum'})\n",
    "\n",
    "    df_sorted_station = df_sorted_station.reset_index(level=['station_unique', 'day_of_week'])\n",
    "\n",
    "    df_busiest = df_sorted_station.groupby('station_unique').sum().sort_values('total_turns',\\\n",
    "                                                                            ascending = False)\n",
    "    \n",
    "    \n",
    "    busiest_stations = df_busiest.index\n",
    "    \n",
    "    plt.figure(figsize=(14,10))\n",
    "    \n",
    "    if custom_stations != None:\n",
    "        for station in custom_stations:\n",
    "            df_subset = df_sorted_station[df_sorted_station.station_unique == station]\n",
    "            daily_mean = (df_subset.total_turns.sum())/df_subset.total_turns.count()\n",
    "            df_subset['deviation_from_daily_mean'] = (df_subset['total_turns']-daily_mean)/daily_mean\n",
    "            plt.plot('day_of_week', 'total_turns', data=df_subset, label=station)\n",
    "    else:\n",
    "        for station in busiest_stations[:num_stations]:\n",
    "            df_subset = df_sorted_station.loc[df_sorted_station['station_unique']==station].copy()\n",
    "            daily_mean = (df_subset.total_turns.sum())/df_subset.total_turns.count()\n",
    "            df_subset['deviation_from_daily_mean'] = (df_subset['total_turns']-daily_mean)/daily_mean\n",
    "            plt.plot('day_of_week', 'total_turns', data=df_subset, label=station)\n",
    "    plt.legend(framealpha=0.25, fontsize='small', bbox_to_anchor=(1.05,1))\n",
    "    plt.xlabel('Day of the Week')\n",
    "    plt.ylabel('Total Turnstile Turns')\n",
    "    plt.title(title, fontsize='x-large')\n",
    "    plt.tight_layout()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def histogram(df_sorted):\n",
    "    import matplotlib.pyplot as plt\n",
    "    %matplotlib inline\n",
    "    df_sorted_station = df_sorted.groupby(['station_unique', 'turnstiles', 'day_of_week'\\\n",
    "                                              ]).agg({'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                                  'total_turns':'sum'})\n",
    "\n",
    "    df_sorted_station = df_sorted_station.reset_index(level='turnstiles').groupby('station_unique').agg({'entries_diff': \\\n",
    "                                                        'sum', 'exits_diff':'sum',\\\n",
    "                                                          'total_turns':'sum'})\n",
    "    plt.figure(figsize=(14,10))\n",
    "    plt.hist((df_sorted_station['total_turns']), bins = 50);\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "def list_busiest(df_sorted, amount=30):\n",
    "    \n",
    "    df_sorted_station = df_sorted.groupby(['station_unique', 'turnstiles', 'day_of_week'\\\n",
    "                                              ]).agg({'entries_diff': 'sum', 'exits_diff':'sum',\\\n",
    "                                                  'total_turns':'sum'})\n",
    "\n",
    "    df_sorted_station = df_sorted_station.reset_index(level='turnstiles').groupby('station_unique').agg({'entries_diff': \\\n",
    "                                                        'sum', 'exits_diff':'sum',\\\n",
    "                                                          'total_turns':'sum'})\n",
    "    return df_sorted_station['total_turns'].sort_values(ascending=False).head(amount)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def plot_traffic_by_time(df_sorted, borough=False):\n",
    "    import matplotlib.pyplot as plt\n",
    "    %matplotlib inline\n",
    "    \n",
    "    if borough:\n",
    "            df_sorted = df_sorted[df_sorted['borough']==borough]\n",
    "\n",
    "    df_times = df_sorted.groupby(['station_unique', 'time_round']).sum()\n",
    "    df_times.reset_index(level=['time_round', 'station_unique'], inplace=True)\n",
    "    df_times.head(20)\n",
    "\n",
    "    for num in range(len(df_times['time_round'])):\n",
    "        if df_times.loc[num, 'time_round'] >4 and df_times.loc[num, 'time_round'] <=12:\n",
    "            df_times.loc[num, 'time_round'] = 'Morning: 4am - 12pm'\n",
    "        elif df_times.loc[num, 'time_round'] >12 and df_times.loc[num, 'time_round'] <=20:\n",
    "            df_times.loc[num, 'time_round'] = 'Afternoon: 12pm - 8pm'\n",
    "        else:\n",
    "            df_times.loc[num, 'time_round'] = 'Night: 8pm - 4am'\n",
    "        \n",
    "    df_busiest = df_times.groupby('station_unique').sum().sort_values('total_turns',\\\n",
    "                                                                            ascending = False)\n",
    "    \n",
    "    \n",
    "    busiest_stations = df_busiest.index\n",
    "\n",
    "    # if custom_stations != None:#     for station in custom_stations:#         df_subset = df_sorted_station[df_sorted_station.station_unique == station]#         daily_mean = (df_subset.total_turns.sum())/df_subset.total_turns.count()#         df_subset['deviation_from_daily_mean'] = (df_subset['total_turns']-daily_mean)/daily_mean#         plt.plot('day_of_week', 'total_turns', data=df_subset, label=station)# else:plt.figure(figsize=[13,10])\n",
    "    plt.figure(figsize=(13,10))\n",
    "    plt.subplot(1, 2, 1)\n",
    "    plt.suptitle('Relative Turnstile Traffic Of Busiest Stations By Time of Day', y=1.0)\n",
    "    for station in busiest_stations[:10]:\n",
    "        df_subset = df_times.loc[df_times['station_unique']==station].groupby('time_round').sum()\n",
    "        weekly_turns = (df_subset.weekly_turns.sum())\n",
    "        df_subset['ratio'] = df_subset['weekly_turns']/weekly_turns\n",
    "        plt.scatter(df_subset.index, df_subset['ratio'],  label=station, s = 50, marker='D')\n",
    "    plt.legend(framealpha=0.5, fontsize='small')\n",
    "    plt.ylabel('Percentage of Total Turnstile Traffic')\n",
    "    plt.title('Top 10 Busiest Stations')\n",
    "\n",
    "    plt.subplot(1,2,2)\n",
    "    for station in busiest_stations[10:20]:\n",
    "        df_subset = df_times.loc[df_times['station_unique']==station].groupby('time_round').sum()\n",
    "        weekly_turns = (df_subset.weekly_turns.sum())\n",
    "        df_subset['ratio'] = df_subset['weekly_turns']/weekly_turns\n",
    "        plt.scatter(df_subset.index, df_subset['ratio'],  label=station, s = 50, marker='D')\n",
    "    plt.legend(framealpha=0.5, fontsize='small')\n",
    "    plt.title('11th - 20th Busiest Stations')\n",
    "    \n",
    "# plot_traffic_by_time(df_sorted, borough='Brooklyn')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}