Skip to content

Commit c3865f7

Browse files
jukentjukent
committed
add scripts for more plotting and nest automate-metrics better
1 parent 2ffd0eb commit c3865f7

7 files changed

+279
-70
lines changed

‎.github/workflows/automate-metrics.yaml

-34
This file was deleted.

‎.github/workflows/get-metrics.py

+177-24
Original file line numberDiff line numberDiff line change
@@ -1,36 +1,44 @@
11
import json
2+
import math
23
import os
3-
import base64
4-
import hashlib
54

5+
import cartopy
6+
import matplotlib.cm as cm
7+
import matplotlib.colors as colors
8+
import matplotlib.pyplot as plt
9+
import numpy as np
610
from google.analytics.data_v1beta import BetaAnalyticsDataClient
7-
from google.analytics.data_v1beta.types import DateRange, Metric, RunReportRequest
11+
from google.analytics.data_v1beta.types import DateRange, Dimension, Metric, RunReportRequest
812

9-
PORTAL_ID = os.environ.get('PORTAL_ID')
10-
FOUNDATIONS_ID = os.environ.get('FOUNDATIONS_ID')
11-
COOKBOOKS_ID = os.environ.get('COOKBOOKS_ID')
13+
PORTAL_ID = os.environ['PORTAL_ID']
14+
FOUNDATIONS_ID = os.environ['FOUNDATIONS_ID']
15+
COOKBOOKS_ID = os.environ['COOKBOOKS_ID']
1216

1317
PRIVATE_KEY_ID = os.environ.get('PRIVATE_KEY_ID')
14-
PRIVATE_KEY = os.environ.get('PRIVATE_KEY').replace('$','\n')
18+
PRIVATE_KEY = os.environ.get('PRIVATE_KEY').replace('$', '\n')
1519

1620
credentials_dict = {
17-
"type": "service_account",
18-
"project_id": "cisl-vast-pythia",
19-
"private_key_id": PRIVATE_KEY_ID,
20-
"private_key": PRIVATE_KEY,
21-
"client_email": "[email protected]",
22-
"client_id": "113402578114110723940",
23-
"auth_uri": "https://accounts.google.com/o/oauth2/auth",
24-
"token_uri": "https://oauth2.googleapis.com/token",
25-
"auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",
26-
"client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com",
27-
"universe_domain": "googleapis.com"
21+
'type': 'service_account',
22+
'project_id': 'cisl-vast-pythia',
23+
'private_key_id': PRIVATE_KEY_ID,
24+
'private_key': PRIVATE_KEY,
25+
'client_email': '[email protected]',
26+
'client_id': '113402578114110723940',
27+
'auth_uri': 'https://accounts.google.com/o/oauth2/auth',
28+
'token_uri': 'https://oauth2.googleapis.com/token',
29+
'auth_provider_x509_cert_url': 'https://www.googleapis.com/oauth2/v1/certs',
30+
'client_x509_cert_url': 'https://www.googleapis.com/robot/v1/metadata/x509/pythia-metrics-api%40cisl-vast-pythia.iam.gserviceaccount.com',
31+
'universe_domain': 'googleapis.com',
2832
}
2933

3034
client = BetaAnalyticsDataClient.from_service_account_info(credentials_dict)
3135

3236

33-
def _run_total_users_report(property_id):
37+
def _format_rounding(value):
38+
return f"{round(value / 1000, 1):.1f}K"
39+
40+
41+
def run_total_users_report(property_id):
3442
request = RunReportRequest(
3543
property=f'properties/{property_id}',
3644
dimensions=[],
@@ -44,18 +52,163 @@ def _run_total_users_report(property_id):
4452
for row in response.rows:
4553
total_users += int(row.metric_values[0].value)
4654

47-
return total_users
55+
return _format_rounding(total_users)
56+
57+
58+
def _run_top_pages_report(property_id):
59+
request = RunReportRequest(
60+
property=f'properties/{property_id}',
61+
dimensions=[Dimension(name='pageTitle')],
62+
date_ranges=[DateRange(start_date='2020-03-31', end_date='today')],
63+
metrics=[Metric(name='screenPageViews')],
64+
)
65+
66+
response = client.run_report(request)
67+
68+
page_views = {}
69+
for row in response.rows:
70+
page = row.dimension_values[0].value
71+
views = int(row.metric_values[0].value)
72+
page_views[page] = views
73+
74+
top_10_pages = sorted(page_views.items(), key=lambda item: item[1], reverse=True)[:10]
75+
top_10_pages_dict = {page: views for page, views in top_10_pages}
76+
77+
return top_10_pages_dict
78+
79+
80+
def plot_top_pages(foundations_id, cookbooks_id):
81+
foundations_page_views = _run_top_pages_report(foundations_id)
82+
foundations_pages = []
83+
foundations_sorted = {k: v for k, v in sorted(foundations_page_views.items(), key=lambda item: item[1])}
84+
foundations_views = foundations_sorted.values()
85+
for key in foundations_sorted:
86+
newkey = key.split('—')[0]
87+
foundations_pages.append(newkey)
88+
89+
cookbooks_page_views = _run_top_pages_report(cookbooks_id)
90+
cookbooks_pages = []
91+
cookbooks_sorted = {k: v for k, v in sorted(cookbooks_page_views.items(), key=lambda item: item[1])}
92+
cookbooks_views = cookbooks_sorted.values()
93+
for key in cookbooks_page_views:
94+
newkey = key.split('—')[0]
95+
cookbooks_pages.insert(0, newkey)
96+
97+
pages = cookbooks_pages + foundations_pages
98+
99+
fig, ax = plt.subplots()
100+
plt.title('Most Popular Pages')
101+
102+
views_max = int(math.ceil(max(foundations_views) / 10000.0)) * 10000
103+
ax.set_xlim([0, views_max])
104+
105+
y = np.arange(10)
106+
y2 = np.arange(11, 21)
107+
y3 = np.append(y, y2)
108+
109+
bar1 = ax.barh(y2, foundations_views, align='center', label='Foundations', color='royalblue')
110+
bar2 = ax.barh(y, cookbooks_views, align='center', label='Cookbooks', color='indianred')
111+
112+
ax.set_yticks(y3, labels=pages)
113+
114+
ax.bar_label(bar1, fmt=_format_rounding)
115+
ax.bar_label(bar2, fmt=_format_rounding)
116+
117+
plt.legend()
118+
plt.savefig('bypage.png', bbox_inches='tight')
119+
120+
121+
def _run_usersXcountry_report(foundations_id):
122+
request = RunReportRequest(
123+
property=f'properties/{foundations_id}',
124+
dimensions=[Dimension(name='country')],
125+
metrics=[Metric(name='activeUsers')],
126+
date_ranges=[DateRange(start_date='2020-03-31', end_date='today')],
127+
)
128+
129+
response = client.run_report(request)
130+
131+
user_by_country = {}
132+
for row in response.rows:
133+
country = row.dimension_values[0].value
134+
users = int(row.metric_values[0].value)
135+
user_by_country[country] = user_by_country.get(country, 0) + users
136+
137+
return user_by_country
138+
139+
140+
def plot_usersXcountry(foundations_id):
141+
users_by_country = _run_usersXcountry_report(foundations_id)
142+
143+
dict_api2cartopy = {
144+
'Tanzania': 'United Republic of Tanzania',
145+
'United States': 'United States of America',
146+
'Congo - Kinshasa': 'Democratic Republic of the Congo',
147+
'Bahamas': 'The Bahamas',
148+
'Timor-Leste': 'East Timor',
149+
'C\u00f4te d\u2019Ivoire': 'Ivory Coast',
150+
'Bosnia & Herzegovina': 'Bosnia and Herzegovina',
151+
'Serbia': 'Republic of Serbia',
152+
'Trinidad & Tobago': 'Trinidad and Tobago',
153+
}
154+
155+
for key in dict_api2cartopy:
156+
users_by_country[dict_api2cartopy[key]] = users_by_country.pop(key)
157+
158+
top_10_countries = sorted(users_by_country.items(), key=lambda item: item[1], reverse=True)[:10]
159+
top_10_text = '\n'.join(
160+
f"{country}: {_format_rounding(value)}" for i, (country, value) in enumerate(top_10_countries)
161+
)
162+
163+
fig = plt.figure(figsize=(10, 4))
164+
ax = plt.axes(projection=cartopy.crs.PlateCarree(), frameon=False)
165+
ax.set_title('Pythia Foundations Unique Users by Country')
166+
167+
shapefile = cartopy.io.shapereader.natural_earth(category='cultural', resolution='110m', name='admin_0_countries')
168+
reader = cartopy.io.shapereader.Reader(shapefile)
169+
countries = reader.records()
170+
171+
colormap = plt.get_cmap('Blues')
172+
colormap.set_extremes(under='grey')
173+
vmax = int(math.ceil(max(users_by_country.values()) / 100.0)) * 100
174+
norm = colors.LogNorm(vmin=1, vmax=vmax)
175+
mappable = cm.ScalarMappable(norm=norm, cmap=colormap)
176+
177+
for country in countries:
178+
country_name = country.attributes['SOVEREIGNT']
179+
if country_name in users_by_country.keys():
180+
facecolor = colormap((users_by_country[country_name] / 105))
181+
182+
ax.add_geometries(
183+
[country.geometry], cartopy.crs.PlateCarree(), facecolor=facecolor, edgecolor='white', linewidth=0.7
184+
)
185+
else:
186+
ax.add_geometries(
187+
[country.geometry], cartopy.crs.PlateCarree(), facecolor='grey', edgecolor='white', linewidth=0.7
188+
)
189+
190+
cax = fig.add_axes([0.1, -0.015, 0.67, 0.03])
191+
fig.colorbar(mappable=mappable, cax=cax, spacing='uniform', orientation='horizontal', extend='min')
192+
193+
props = dict(boxstyle='round', facecolor='white', edgecolor='white')
194+
ax.text(1.01, 0.5, top_10_text, transform=ax.transAxes, fontsize=9, verticalalignment='center', bbox=props)
195+
196+
plt.tight_layout()
197+
plt.savefig('bycountry.png', bbox_inches='tight')
48198

49199

50200
def get_metrics():
51201
metrics_dict = {}
52-
metrics_dict['Portal'] = _run_total_users_report(str(PORTAL_ID))
53-
metrics_dict['Foundations'] = _run_total_users_report(str(FOUNDATIONS_ID))
54-
metrics_dict['Cookbooks'] = _run_total_users_report(str(COOKBOOKS_ID))
55-
202+
metrics_dict['Portal'] = run_total_users_report(str(PORTAL_ID))
203+
metrics_dict['Foundations'] = run_total_users_report(str(FOUNDATIONS_ID))
204+
metrics_dict['Cookbooks'] = run_total_users_report(str(COOKBOOKS_ID))
56205
with open('user_metrics.json', 'w') as outfile:
57206
json.dump(metrics_dict, outfile)
58207

208+
plot_top_pages(str(FOUNDATIONS_ID), str(COOKBOOKS_ID))
209+
210+
plot_usersXcountry(str(FOUNDATIONS_ID))
211+
59212

60213
if __name__ == '__main__':
61214
get_metrics()

‎.github/workflows/nightly-build.yaml

+23-2
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,30 @@ on:
55
schedule:
66
- cron: '0 0 * * *' # Daily “At 00:00”
77

8+
env:
9+
PORTAL_ID: ${{ secrets.PORTAL_ID }}
10+
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
11+
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
12+
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
13+
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
14+
815
jobs:
9-
update-metrics:
10-
uses: ./.github/workflows/automate-metrics.yaml
16+
automate-metrics:
17+
runs-on: macos-latest
18+
steps:
19+
- uses: actions/checkout@v3
20+
- name: Automate Metrics
21+
run: |
22+
python -m venv analytics-api
23+
source analytics-api/bin/activate
24+
pip install google-analytics-data
25+
conda install cartopy matplotlib
26+
27+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
28+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py
29+
30+
python get-metrics.py
31+
python write-metrics-md.py
1132
1233
build:
1334
if: ${{ github.repository_owner == 'ProjectPythia' }}

‎.github/workflows/publish-site.yaml

+23-2
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,30 @@ on:
77
- main
88
workflow_dispatch:
99

10+
env:
11+
PORTAL_ID: ${{ secrets.PORTAL_ID }}
12+
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
13+
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
14+
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
15+
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
16+
1017
jobs:
11-
update-metrics:
12-
uses: ./.github/workflows/automate-metrics.yaml
18+
automate-metrics:
19+
runs-on: macos-latest
20+
steps:
21+
- uses: actions/checkout@v3
22+
- name: Automate Metrics
23+
run: |
24+
python -m venv analytics-api
25+
source analytics-api/bin/activate
26+
pip install google-analytics-data
27+
conda install cartopy matplotlib
28+
29+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
30+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py
31+
32+
python get-metrics.py
33+
python write-metrics-md.py
1334
1435
build:
1536
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main

‎.github/workflows/trigger-preview.yaml

+23-2
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,30 @@ on:
77
- requested
88
- completed
99

10+
env:
11+
PORTAL_ID: ${{ secrets.PORTAL_ID }}
12+
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
13+
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
14+
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
15+
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
16+
1017
jobs:
11-
update-metrics:
12-
uses: ./.github/workflows/automate-metrics.yaml
18+
automate-metrics:
19+
runs-on: macos-latest
20+
steps:
21+
- uses: actions/checkout@v3
22+
- name: Automate Metrics
23+
run: |
24+
python -m venv analytics-api
25+
source analytics-api/bin/activate
26+
pip install google-analytics-data
27+
conda install cartopy matplotlib
28+
29+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
30+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py
31+
32+
python get-metrics.py
33+
python write-metrics-md.py
1334
1435
find-pull-request:
1536
uses: ProjectPythia/cookbook-actions/.github/workflows/find-pull-request.yaml@main

‎.github/workflows/trigger-site-build.yaml

+23-2
Original file line numberDiff line numberDiff line change
@@ -2,9 +2,30 @@ name: trigger-site-build
22
on:
33
pull_request:
44

5+
env:
6+
PORTAL_ID: ${{ secrets.PORTAL_ID }}
7+
FOUNDATIONS_ID: ${{ secrets.FOUNDATIONS_ID }}
8+
COOKBOOKS_ID: ${{ secrets.COOKBOOKS_ID }}
9+
PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
10+
PRIVATE_KEY_ID: ${{ secrets.PRIVATE_KEY_ID }}
11+
512
jobs:
6-
update-metrics:
7-
uses: ./.github/workflows/automate-metrics.yaml
13+
automate-metrics:
14+
runs-on: macos-latest
15+
steps:
16+
- uses: actions/checkout@v3
17+
- name: Automate Metrics
18+
run: |
19+
python -m venv analytics-api
20+
source analytics-api/bin/activate
21+
pip install google-analytics-data
22+
conda install cartopy matplotlib
23+
24+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/get-metrics.py
25+
curl -O https://raw.githubusercontent.com/jukent/projectpythia.github.io/main/.github/workflows/write-metrics-md.py
26+
27+
python get-metrics.py
28+
python write-metrics-md.py
829
930
build:
1031
uses: ProjectPythia/cookbook-actions/.github/workflows/build-book.yaml@main

0 commit comments

Comments
 (0)