cleaning up and fixing tests

Author: aysim319

claims_hosp/delphi_claims_hosp/run.py

@@ -56,10 +56,10 @@ def run_module(params, logger=None):
                 adjustments (False).
     """
     start_time = time.time()
-    issue_date_str = params.get("patch", {}).get("current_issue", None)
-    issue_date = datetime.strptime(issue_date_str, "%Y-%m-%d")
     # safety check for patch parameters exists in file, but not running custom runs/patches
     custom_run_flag = False if not params["common"].get("custom_run", False) else params["common"]["custom_run"]
+    issue_date_str = params.get("patch", {}).get("current_issue", None)
+    issue_date = datetime.strptime(issue_date_str, "%Y-%m-%d") if issue_date_str else None
     if not logger:
         logger = get_structured_logger(
             __name__,

claims_hosp/tests/conftest.py

@@ -19,9 +19,9 @@
 def params():
     params = {
         "common": {
-            "export_dir": f"{TEST_DIR}/receiving",
+            "export_dir": f"{TEST_DIR}/retrieve_files",
             "log_filename": f"{TEST_DIR}/test.log",
-            "custom_run": True,
+            "custom_run": False,
         },
         "indicator": {
             "drop_date": None,
@@ -58,9 +58,10 @@ def params():
 @pytest.fixture
 def params_w_patch(params):
     params_copy = copy.deepcopy(params)
+    params_copy["common"]["custom_run"] = True
     params_copy["patch"] = {
-            "start_issue": "2020-11-07",
-            "end_issue": "2020-11-07",
+            "start_issue": "2020-06-12",
+            "end_issue": "2020-06-12",
             "patch_dir": "./patch_dir"
         }
     return params_copy

claims_hosp/tests/retrieve_files/.gitignore

@@ -4,10 +4,13 @@
 from unittest.mock import patch as mock_patch, MagicMock
 import os
 import shutil
+
+from delphi_claims_hosp.download_claims_ftp_files import change_date_format
+
 from conftest import TEST_DIR
 
 from delphi_claims_hosp.patch import patch
-from delphi_claims_hosp.backfill import merge_existing_backfill_files
+from delphi_claims_hosp.backfill import merge_existing_backfill_files, merge_backfill_file
 
 class TestPatchModule:
 
@@ -16,24 +19,28 @@ def test_patch(self, params_w_patch):
             mock_patch('delphi_claims_hosp.patch.read_params') as mock_read_params, \
             mock_patch('delphi_claims_hosp.download_claims_ftp_files.paramiko.SSHClient') as mock_ssh_client, \
             mock_patch('delphi_claims_hosp.download_claims_ftp_files.path.exists', return_value=False), \
-            mock_patch('delphi_claims_hosp.backfill.merge_existing_backfill_files') as mock_backfill:
+            mock_patch('delphi_claims_hosp.run.merge_existing_backfill_files') as mock_patch_backfill, \
+            mock_patch('delphi_claims_hosp.run.merge_backfill_file') as mock_backfill:
             mock_ssh_client_instance = MagicMock()
             mock_ssh_client.return_value = mock_ssh_client_instance
             mock_sftp = MagicMock()
             mock_ssh_client_instance.open_sftp.return_value = mock_sftp
-            mock_sftp.listdir_attr.return_value = [MagicMock(filename="SYNEDI_AGG_INPATIENT_11062020_1451CDT.csv.gz")]
+            mock_sftp.listdir_attr.return_value = [MagicMock(filename=change_date_format("SYNEDI_AGG_INPATIENT_06112020_1451CDT.csv.gz"))]
             def mock_get(*args, **kwargs):
-                src = Path(f"{TEST_DIR}/test_data/{args[0]}")
-                dst = Path(f"{TEST_DIR}/receiving/{args[0]}")
+                file = change_date_format(args[0])
+                src = Path(f"{TEST_DIR}/test_data/{file}")
+                dst = Path(f"{TEST_DIR}/receiving/{file}")
                 shutil.copyfile(src, dst)
             mock_sftp.get.side_effect = mock_get
 
             mock_read_params.return_value = params_w_patch
-            mock_backfill.side_effect = merge_existing_backfill_files
+            mock_patch_backfill.side_effect = merge_existing_backfill_files
+            mock_backfill.side_effect = merge_backfill_file
 
             patch()
 
-            # assert mock_backfill.assert_called()
+            assert mock_patch_backfill.call_count == 1
+            assert mock_backfill.call_count == 0
 
             issue_date = params_w_patch["patch"]["start_issue"].replace("-", "")
             assert os.path.isdir(f'{TEST_DIR}/patch_dir/issue_{issue_date}/hospital-admissions')

claims_hosp/tests/test_patch.py

@@ -1,13 +1,43 @@
-import logging
-from os import listdir
-from os.path import join
-from itertools import product
+from pathlib import Path
+from unittest.mock import patch, MagicMock
+import shutil
 
-import pandas as pd
 import pytest
+from delphi_claims_hosp.download_claims_ftp_files import change_date_format
+
+from delphi_claims_hosp.run import run_module
+from delphi_claims_hosp.backfill import merge_existing_backfill_files, merge_backfill_file
+from freezegun import freeze_time
 
 from conftest import TEST_DIR
 class TestRun:
-    @pytest.mark.freeze_time("2020-11-07")
-    def test_output_files_exist(self, run_as_module):
-        pass
+    @freeze_time("2020-06-11 20:00:00")
+    def test_output_files(self, params):
+        with patch('delphi_claims_hosp.patch.get_structured_logger'), \
+            patch('delphi_claims_hosp.download_claims_ftp_files.paramiko.SSHClient') as mock_ssh_client, \
+            patch('delphi_claims_hosp.download_claims_ftp_files.path.exists', return_value=False), \
+            patch('delphi_claims_hosp.run.merge_existing_backfill_files') as mock_patch_backfill, \
+            patch('delphi_claims_hosp.run.merge_backfill_file') as mock_backfill:
+
+            mock_ssh_client_instance = MagicMock()
+            mock_ssh_client.return_value = mock_ssh_client_instance
+            mock_sftp = MagicMock()
+            mock_ssh_client_instance.open_sftp.return_value = mock_sftp
+            mock_sftp.listdir_attr.return_value = [MagicMock(filename="SYNEDI_AGG_INPATIENT_20200611_1451CDT.csv.gz")]
+            def mock_get(*args, **kwargs):
+                file = change_date_format(args[0])
+                src = Path(f"{TEST_DIR}/test_data/{file}")
+                dst = Path(f"{TEST_DIR}/receiving/{file}")
+                shutil.copyfile(src, dst)
+            mock_sftp.get.side_effect = mock_get
+
+            mock_patch_backfill.side_effect = merge_existing_backfill_files
+            mock_backfill.side_effect = merge_backfill_file
+
+            run_module(params)
+
+            assert mock_patch_backfill.call_count == 0
+            assert mock_backfill.call_count == 1
+
+        # Clean up the created directories after the test
+        shutil.rmtree(params["common"]["export_dir"])