Add in archived dashboards, and calculate start year from data

Author: cchuong

src/acquisition/rvdss/constants.py

@@ -76,6 +76,7 @@
 SEASON_BASE_URL = "https://www.canada.ca"
 ALTERNATIVE_SEASON_BASE_URL = "www.phac-aspc.gc.ca/bid-bmi/dsd-dsm/rvdi-divr/"
 HISTORIC_SEASON_REPORTS_URL = SEASON_BASE_URL+"/en/public-health/services/surveillance/respiratory-virus-detections-canada/{year_range}.html"
+DASHBOARD_ARCHIVED_DATES_URL= "https://health-infobase.canada.ca/src/js/respiratory-virus-detections/ArchiveData.json"
 
 # Each URL created here points to a list of all data reports made during that
 # season, e.g.
@@ -103,10 +104,11 @@
 DASHBOARD_UPDATE_DATE_FILE = "RVD_UpdateDate.csv"
 DASHBOARD_DATA_FILE = "RVD_WeeklyData.csv"
 
+
 RESP_DETECTIONS_OUTPUT_FILE = "respiratory_detections.csv"
 POSITIVE_TESTS_OUTPUT_FILE = "positive_tests.csv"
 COUNTS_OUTPUT_FILE = "number_of_detections.csv"
 
-LAST_WEEK_OF_YEAR = 35
+FIRST_WEEK_OF_YEAR = 35
 
 NOW = datetime.datetime.now()

src/acquisition/rvdss/pull_historic.py

@@ -13,15 +13,16 @@
 from epiweeks import Week
 from datetime import datetime, timedelta
 import math
+from pandas.io.json import json_normalize
 
 from delphi.epidata.acquisition.rvdss.constants import (
         DASHBOARD_BASE_URLS_2023_2024_SEASON, HISTORIC_SEASON_URLS,
-        ALTERNATIVE_SEASON_BASE_URL, SEASON_BASE_URL, LAST_WEEK_OF_YEAR,
-        RESP_DETECTIONS_OUTPUT_FILE, POSITIVE_TESTS_OUTPUT_FILE
+        ALTERNATIVE_SEASON_BASE_URL, SEASON_BASE_URL, FIRST_WEEK_OF_YEAR,
+        RESP_DETECTIONS_OUTPUT_FILE, POSITIVE_TESTS_OUTPUT_FILE,DASHBOARD_ARCHIVED_DATES_URL
     )
 from delphi.epidata.acquisition.rvdss.utils import (
         abbreviate_virus, abbreviate_geo, create_geo_types, check_date_format,
-        get_revised_data, get_weekly_data
+        get_revised_data, get_weekly_data, fetch_dashboard_data
     )
  #%% Functions
 
@@ -78,7 +79,7 @@ def get_report_date(week,start_year,epi=False):
     epi - if True, return the date in cdc format (yearweek)
     
     """
-    if week < LAST_WEEK_OF_YEAR: 
+    if week < FIRST_WEEK_OF_YEAR: 
         year=int(start_year)+1
     else:
         year=int(start_year)
@@ -523,7 +524,7 @@ def fetch_one_season_from_report(url):
                positive_tables.append(pos_table)
 
         # create path to save files
-        path = "season_" + season[0]+"_"+season[1]
+        #path = "season_" + season[0]+"_"+season[1]
 
         # combine all the positive tables
         combined_positive_tables=pd.concat(positive_tables,axis=1)
@@ -548,9 +549,13 @@ def fetch_one_season_from_report(url):
         "count": all_number_tables,
     }
 
-def fetch_archived_dashboard_urls():
-    ## TODO: paste in Christine's code for scraping this list https://health-infobase.canada.ca/respiratory-virus-detections/archive.html
-    pass
+def fetch_archived_dashboard_urls(archive_url):
+    r=requests.get(archive_url)
+    values=r.json()
+    data=json_normalize(values)
+    archived_dates = data[data["lang"]=="en"]
+    return(archived_dates)
+
 
 def fetch_report_data():
     # Scrape each season.
@@ -560,7 +565,7 @@ def fetch_report_data():
 
 def fetch_historical_dashboard_data():
     # Update the end of the 2023-2024 season with the dashboard data
-    included_urls = fetch_archived_dashboard_urls()
-    dict_list = [fetch_dashboard_data(url, 2023) for url in included_urls]
+    included_urls = fetch_archived_dashboard_urls(DASHBOARD_ARCHIVED_DATES_URL)
+    dict_list = [fetch_dashboard_data(url) for url in included_urls]
 
     return dict_list

src/acquisition/rvdss/utils.py

@@ -9,7 +9,7 @@
 import string
 
 from delphi.epidata.acquisition.rvdss.constants import (
-        VIRUSES, GEOS, REGIONS, NATION, LAST_WEEK_OF_YEAR,
+        VIRUSES, GEOS, REGIONS, NATION, FIRST_WEEK_OF_YEAR,
         DASHBOARD_UPDATE_DATE_FILE, DASHBOARD_DATA_FILE
     )
 
@@ -105,7 +105,7 @@ def get_revised_data(base_url,headers,update_date):
     return(df)
 
 ## TODO: the `start_year` arg is making calling this complicated. If we know that LAST_WEEK_OF_YEAR (really, of the season) is always 35, then we should be able to derive `start_year` from `update_date`.
-def get_weekly_data(base_url,start_year,headers,update_date):
+def get_weekly_data(base_url,headers,update_date):
     # Get current week and year
     summary_url =  base_url + "RVD_SummaryText.csv"
     summary_url_response = requests.get(summary_url, headers=headers)
@@ -114,11 +114,14 @@ def get_weekly_data(base_url,start_year,headers,update_date):
     week_df = summary_df[(summary_df['Section'] == "summary") & (summary_df['Type']=="title")]
     week_string = week_df.iloc[0]['Text'].lower()
     current_week = int(re.search("week (.+?) ", week_string).group(1))
+    current_year= int(re.search("20\d{2}", week_string).group(0))
 
-    if current_week < LAST_WEEK_OF_YEAR:
+    '''
+    if current_week < FIRST_WEEK_OF_YEAR:
         current_year = start_year+1
     else:
         current_year = start_year
+    '''
 
     current_epiweek= Week(current_year,current_week)
 
@@ -150,15 +153,15 @@ def get_weekly_data(base_url,start_year,headers,update_date):
 
     return(df_weekly.set_index(['epiweek', 'time_value', 'issue', 'geo_type', 'geo_value']))
 
-def fetch_dashboard_data(url, start_year):
+def fetch_dashboard_data(url):
     """Get data from current or archived dashboard"""
     headers = {
         'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36'
     }
 
     update_date = get_dashboard_update_date(url, headers)
 
-    weekly_data = get_weekly_data(url,start_year,headers,update_date)
+    weekly_data = get_weekly_data(url,headers,update_date)
     positive_data = get_revised_data(url,headers,update_date)
 
     ## TODO: how to "weekly" and "positive" correspond to the dict keys ("respiratory_detection", "positive", "count") from historical reports? Need to make sure keys used here are from the same set.