Add hsa_nci geo resolution + Adjust csv regex patterns

Author: minhkhul

src/acquisition/covidcast/csv_importer.py

@@ -52,18 +52,21 @@ class CsvRowValue:
 class CsvImporter:
   """Finds and parses covidcast CSV files."""
 
+  # set of allowed resolutions (aka "geo_type")
+  GEOGRAPHIC_RESOLUTIONS = {'county', 'hrr', 'msa', 'dma', 'state', 'hhs', 'nation', "hsa", "hsa_nci"}
+
+  # regex pattern for matching geo types, note: sort longer string first to avoid wrong substring matches
+  geo_types_pattern = "|".join(sorted(GEOGRAPHIC_RESOLUTIONS, key=len, reverse=True))
+
   # .../source/yyyymmdd_geo_signal.csv
-  PATTERN_DAILY = re.compile(r'^.*/([^/]*)/(\d{8})_(\w+?)_(\w+)\.csv$')
+  PATTERN_DAILY = re.compile(rf'^.*/([^/]*)/(\d{{8}})_({geo_types_pattern})_(\w+)\.csv$')
 
   # .../source/weekly_yyyyww_geo_signal.csv
-  PATTERN_WEEKLY = re.compile(r'^.*/([^/]*)/weekly_(\d{6})_(\w+?)_(\w+)\.csv$')
+  PATTERN_WEEKLY = re.compile(rf'^.*/([^/]*)/weekly_(\d{{6}})_({geo_types_pattern})_(\w+)\.csv$')
 
   # .../issue_yyyymmdd
   PATTERN_ISSUE_DIR = re.compile(r'^.*/([^/]*)/issue_(\d{8})$')
 
-  # set of allowed resolutions (aka "geo_type")
-  GEOGRAPHIC_RESOLUTIONS = {'county', 'hrr', 'msa', 'dma', 'state', 'hhs', 'nation'}
-
   # set of required CSV columns
   REQUIRED_COLUMNS = {'geo_id', 'val', 'se', 'sample_size'}
 

tests/acquisition/covidcast/test_csv_importer.py

@@ -96,6 +96,14 @@ def test_find_csv_files(self, mock_glob: MagicMock):
       path_prefix + 'invalid/20200418_province_c.csv',
       # ignored
       path_prefix + 'ignored/README.md',
+      # valid hsa day
+      path_prefix + 'valid/20200408_hsa_sig.csv',
+      # valid hsa_nci day
+      path_prefix + 'valid/20200408_hsa_nci_sig.csv',
+      # valid hsa_nci week
+      path_prefix + 'valid/weekly_202015_hsa_nci_sig.csv',
+      # valid hsa week
+      path_prefix + 'valid/weekly_202015_hsa_sig.csv',
     ]
     mock_glob.return_value = glob_paths
 
@@ -113,6 +121,10 @@ def test_find_csv_files(self, mock_glob: MagicMock):
       (glob_paths[5], None),
       (glob_paths[6], None),
       (glob_paths[7], None),
+      (glob_paths[8], PathDetails(expected_issue_day, (date.today() - date(year=time_value_day // 10000, month=(time_value_day // 100) % 100, day=time_value_day % 100)).days, 'valid', 'sig', 'day', time_value_day, 'hsa')),
+      (glob_paths[9], PathDetails(expected_issue_day, (date.today() - date(year=time_value_day // 10000, month=(time_value_day // 100) % 100, day=time_value_day % 100)).days, 'valid', 'sig', 'day', time_value_day, 'hsa_nci')),
+      (glob_paths[10], PathDetails(expected_issue_week, delta_epiweeks(202015, expected_issue_week), 'valid', 'sig', 'week', 202015, 'hsa_nci')),
+      (glob_paths[11], PathDetails(expected_issue_week, delta_epiweeks(202015, expected_issue_week), 'valid', 'sig', 'week', 202015, 'hsa')),
     ])
     self.assertEqual(found, expected)