Add comments to metrics code and add error handling (#413)

* Update write-metrics-md.py with comments

* Update get-metrics.py with comments and error handling

Author: jukent

.github/workflows/get-metrics.py

@@ -3,6 +3,7 @@
 import os
 
 import cartopy
+import google
 import matplotlib
 import matplotlib.cm as cm
 import matplotlib.colors as colors
@@ -11,10 +12,13 @@
 from google.analytics.data_v1beta import BetaAnalyticsDataClient
 from google.analytics.data_v1beta.types import DateRange, Dimension, Metric, RunReportRequest
 
+# Project ID Numbers
 PORTAL_ID = '266784902'
 FOUNDATIONS_ID = '281776420'
 COOKBOOKS_ID = '324070631'
 
+
+# Access Secrets
 PRIVATE_KEY_ID = os.environ.get('PRIVATE_KEY_ID')
 PRIVATE_KEY = os.environ.get('PRIVATE_KEY')
 
@@ -32,16 +36,27 @@
     'universe_domain': 'googleapis.com',
 }
 
-client = BetaAnalyticsDataClient.from_service_account_info(credentials_dict)
+try:
+    client = BetaAnalyticsDataClient.from_service_account_info(credentials_dict)
+except google.auth.exceptions.MalformedError as e:
+    print('Malformed Error:', repr(e))
+    # Insight into reason for failure without exposing secret key
+    print('Length of PRIVATE_KEY:', len(PRIVATE_KEY))  # 0: Secret not found, ~734: Secret malformed
 
-pre_project_date = '2020-03-31'  # random date before project start
+pre_project_date = '2020-03-31'  # Random date before project start
 
 
 def _format_rounding(value):
+    """
+    Helper function for rounding string displays. 1,232 -> 1.2K
+    """
     return f'{round(value / 1000, 1):.1f}K'
 
 
 def _run_total_users_report(property_id):
+    """
+    Function for requesting cumulative active users from a project since project start.
+    """
     request = RunReportRequest(
         property=f'properties/{property_id}',
         dimensions=[],
@@ -58,16 +73,24 @@ def _run_total_users_report(property_id):
 
 
 def get_total_users(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
+    """
+    Function for taking cumulative active users from each project and dumping it into a JSON with the current datetime.
+    """
     metrics_dict = {}
     metrics_dict['Now'] = str(datetime.datetime.now())
     metrics_dict['Portal'] = _run_total_users_report(PORTAL_ID)
     metrics_dict['Foundations'] = _run_total_users_report(FOUNDATIONS_ID)
     metrics_dict['Cookbooks'] = _run_total_users_report(COOKBOOKS_ID)
+
+    # Save to JSON, Remember Action is called from root directory
     with open('portal/metrics/user_metrics.json', 'w') as outfile:
         json.dump(metrics_dict, outfile)
 
 
 def _run_active_users_this_year(property_id):
+    """
+    Function for requesting active users by day from a project since year start.
+    """
     current_year = datetime.datetime.now().year
     start_date = f'{current_year}-01-01'
 
@@ -87,14 +110,19 @@ def _run_active_users_this_year(property_id):
         dates.append(date)
         user_counts.append(int(row.metric_values[0].value))
 
+    # Days need to be sorted chronologically
     return zip(*sorted(zip(dates, user_counts), key=lambda x: x[0]))
 
 
 def plot_projects_this_year(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
+    """
+    Function for taking year-to-date active users by day and plotting it for each project.
+    """
     portal_dates, portal_users = _run_active_users_this_year(PORTAL_ID)
     foundations_dates, foundations_users = _run_active_users_this_year(FOUNDATIONS_ID)
     cookbooks_dates, cookbooks_users = _run_active_users_this_year(COOKBOOKS_ID)
 
+    # Plotting code
     plt.figure(figsize=(10, 5.5))
     plt.title('Year-to-Date Pythia Active Users', fontsize=15)
 
@@ -109,6 +137,9 @@ def plot_projects_this_year(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
 
 
 def _run_top_pages_report(property_id):
+    """
+    Function for requesting top 5 pages from a project.
+    """
     request = RunReportRequest(
         property=f'properties/{property_id}',
         dimensions=[Dimension(name='pageTitle')],
@@ -123,48 +154,58 @@ def _run_top_pages_report(property_id):
         views = int(row.metric_values[0].value)
         views_dict[page] = views
 
+    # Sort by views and grab the top 5
     top_pages = sorted(views_dict.items(), key=lambda item: item[1], reverse=True)[:5]
+    # String manipulation on page titles "Cartopy - Pythia Foundations" -> "Cartopy"
     pages = [page.split('—')[0] for page, _ in top_pages]
     views = [views for _, views in top_pages]
 
+    #  Reverse order of lists, so they'll plot with most visited page on top (i.e. last)
     return pages[::-1], views[::-1]
 
 
 def plot_top_pages(PORTAL_ID, FOUNDATIONS_ID, COOKBOOKS_ID):
+    """
+    Function that takes the top 5 viewed pages for all 3 projects and plot them on a histogram.
+    """
     portal_pages, portal_views = _run_top_pages_report(PORTAL_ID)
     foundations_pages, foundations_views = _run_top_pages_report(FOUNDATIONS_ID)
     cookbooks_pages, cookbooks_views = _run_top_pages_report(COOKBOOKS_ID)
 
-    pages = cookbooks_pages + foundations_pages + portal_pages
-
+    # Plotting code
     fig, ax = plt.subplots(figsize=(10, 5.5))
     plt.title('All-Time Top Pages', fontsize=15)
 
-    y = np.arange(5)
-    y2 = np.arange(6, 11)
-    y3 = np.arange(12, 17)
-    y4 = np.append(y, y2)
-    y4 = np.append(y4, y3)
+    y = np.arange(5)  # 0-4 for Cookbooks
+    y2 = np.arange(6, 11)  # 6-10 for Foundations
+    y3 = np.arange(12, 17)  # 12-16 for Portal
 
     bar1 = ax.barh(y3, portal_views, align='center', label='Portal', color='purple')
     bar2 = ax.barh(y2, foundations_views, align='center', label='Foundations', color='royalblue')
     bar3 = ax.barh(y, cookbooks_views, align='center', label='Cookbooks', color='indianred')
 
+    y4 = np.append(y, y2)
+    y4 = np.append(y4, y3)  # 0-4,6-19,12-6 for page labels to have a gap between projects
+    pages = cookbooks_pages + foundations_pages + portal_pages  # List of all pages
     ax.set_yticks(y4, labels=pages, fontsize=12)
 
+    # Adds round-formatted views label to end of each bar
     ax.bar_label(bar1, fmt=_format_rounding, padding=5, fontsize=10)
     ax.bar_label(bar2, fmt=_format_rounding, padding=5, fontsize=10)
     ax.bar_label(bar3, fmt=_format_rounding, padding=5, fontsize=10)
 
     ax.set_xscale('log')
-    ax.set_xlim([10, 10**5])
+    ax.set_xlim([10, 10**5])  # set_xlim must be after setting xscale to log
     ax.set_xlabel('Page Views', fontsize=12)
 
     plt.legend(fontsize=12, loc='lower right')
     plt.savefig('portal/metrics/toppages.png', bbox_inches='tight')
 
 
 def _run_usersXcountry_report(property_id):
+    """
+    Function for requesting users by country for a project.
+    """
     request = RunReportRequest(
         property=f'properties/{property_id}',
         dimensions=[Dimension(name='country')],
@@ -183,6 +224,9 @@ def _run_usersXcountry_report(property_id):
 
 
 def plot_usersXcountry(FOUNDATIONS_ID):
+    """
+    Function for taking users by country for Pythia Foundations and plotting them on a map.
+    """
     users_by_country = _run_usersXcountry_report(FOUNDATIONS_ID)
 
     # Google API Country names do not match Cartopy Country Shapefile names
@@ -201,11 +245,13 @@ def plot_usersXcountry(FOUNDATIONS_ID):
     for key in dict_api2cartopy:
         users_by_country[dict_api2cartopy[key]] = users_by_country.pop(key)
 
+    # Sort by views and grab the top 10 countries for a text box
     top_10_countries = sorted(users_by_country.items(), key=lambda item: item[1], reverse=True)[:10]
     top_10_text = '\n'.join(
         f'{country}: {_format_rounding(value)}' for i, (country, value) in enumerate(top_10_countries)
     )
 
+    # Plotting code
     fig = plt.figure(figsize=(10, 4))
     ax = plt.axes(projection=cartopy.crs.PlateCarree(), frameon=False)
     ax.set_title('Pythia Foundations Users by Country', fontsize=15)
@@ -215,12 +261,13 @@ def plot_usersXcountry(FOUNDATIONS_ID):
     countries = reader.records()
 
     colormap = plt.get_cmap('Blues')
-    newcmp = colors.ListedColormap(colormap(np.linspace(0.2, 1, 128)))
+    newcmp = colors.ListedColormap(colormap(np.linspace(0.2, 1, 128)))  # Truncate colormap to remove white hues
     newcmp.set_extremes(under='grey')
 
-    norm = colors.LogNorm(vmin=1, vmax=max(users_by_country.values()))
+    norm = colors.LogNorm(vmin=1, vmax=max(users_by_country.values()))  # Plot on log scale
     mappable = cm.ScalarMappable(norm=norm, cmap=newcmp)
 
+    # Loop through countries and plot their color
     for country in countries:
         country_name = country.attributes['SOVEREIGNT']
         if country_name in users_by_country.keys():
@@ -238,10 +285,12 @@ def plot_usersXcountry(FOUNDATIONS_ID):
                 [country.geometry], cartopy.crs.PlateCarree(), facecolor='grey', edgecolor='white', linewidth=0.7
             )
 
+    # Add colorbar
     cax = fig.add_axes([0.1, -0.015, 0.67, 0.03])
     cbar = fig.colorbar(mappable=mappable, cax=cax, spacing='uniform', orientation='horizontal', extend='min')
     cbar.set_label('Unique Users')
 
+    # Add top 10 countries text
     props = dict(boxstyle='round', facecolor='white', edgecolor='white')
     ax.text(1.01, 0.5, top_10_text, transform=ax.transAxes, fontsize=12, verticalalignment='center', bbox=props)
 

.github/workflows/write-metrics-md.py

@@ -2,6 +2,9 @@
 
 
 def process_user_data(json_file, top_pages, this_year, map, md_file):
+    """
+    Function for writing portal/metrics.md from saved files output by get-metrics.py
+    """
     with open(json_file, 'r') as f:
         user_data = json.load(f)
 
@@ -12,6 +15,7 @@ def process_user_data(json_file, top_pages, this_year, map, md_file):
         user_data.pop('Now')
         f.write('\n\n')
 
+        # Markdown table
         headers = '| Project | Users |'
         separator = '| ' + ' | '.join(['-----'] * 2) + ' |'
         rows = []
@@ -21,6 +25,7 @@ def process_user_data(json_file, top_pages, this_year, map, md_file):
         f.write(table)
         f.write('\n\n')
 
+        # Add plots
         f.write(f'![Users this Year]({this_year})\n\n')
         f.write(f'![Top Pages]({top_pages})\n\n')
         f.write(f'![Users by Country]({map})\n\n')
@@ -29,9 +34,12 @@ def process_user_data(json_file, top_pages, this_year, map, md_file):
 
 
 if __name__ == '__main__':
-    json_file = 'portal/metrics/user_metrics.json'
+    json_file = 'portal/metrics/user_metrics.json'  # Accessed from root repository
+
+    # HTML is built from within `portal/` directory, so paths differ from json file.
     top_pages = 'metrics/toppages.png'
     this_year = 'metrics/thisyear.png'
     map = 'metrics/bycountry.png'
-    md_file = 'portal/metrics.md'
+
+    md_file = 'portal/metrics.md'  # Written from root repository
     process_user_data(json_file, top_pages, this_year, map, md_file)