Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CNDE-2226: Bmird_strep_pneumo datamart postprocessing proc #233

Merged
merged 9 commits into from
Mar 12, 2025
Merged

CNDE-2226: Bmird_strep_pneumo datamart postprocessing proc #233

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
a5dd26c
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 10, 2025
9e02f2d
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 10, 2025
a0b96a5
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 11, 2025
abc294f
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 11, 2025
fe77a1a
Merge branch 'main' into CNDE-2226
johneipe Mar 12, 2025
4dd6135
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 12, 2025
ed22479
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 12, 2025
859da98
Merge branch 'main' into CNDE-2226
johneipe Mar 12, 2025
b194225
Bmird_strep_pneumo datamart postprocessing proc
johneipe Mar 12, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 5 additions & 5 deletions db/upgrade/rdb_modern/routines/020-sp_nrt_case_count_postprocessing.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ BEGIN TRY
SELECT @ROWCOUNT_NO = @@ROWCOUNT;
INSERT INTO dbo.[JOB_FLOW_LOG]
(BATCH_ID,[DATAFLOW_NAME],[PACKAGE_NAME] ,[STATUS_TYPE],[STEP_NUMBER],[STEP_NAME],[ROW_COUNT])
VALUES(@BATCH_ID,'CASE_COUNT','CASE_COUNT','START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);
VALUES(@BATCH_ID,@dataflow_name, @package_name,'START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);

SET @PROC_STEP_NO = @PROC_STEP_NO + 1;
SET @PROC_STEP_NAME = ' GENERATING Case Count Table - Insert';
Expand Down Expand Up @@ -159,7 +159,7 @@ BEGIN TRY
SELECT @ROWCOUNT_NO = @@ROWCOUNT;
INSERT INTO dbo.[JOB_FLOW_LOG]
(BATCH_ID,[DATAFLOW_NAME],[PACKAGE_NAME] ,[STATUS_TYPE],[STEP_NUMBER],[STEP_NAME],[ROW_COUNT])
VALUES(@BATCH_ID,'CASE_COUNT','CASE_COUNT','START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);
VALUES(@BATCH_ID,@dataflow_name, @package_name,'START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);

COMMIT TRANSACTION;

Expand All @@ -171,7 +171,7 @@ BEGIN TRY
INSERT INTO dbo.[job_flow_log]
(batch_id,[Dataflow_Name],[package_Name],[Status_Type] ,[step_number],[step_name],[row_count])
VALUES
(@batch_id,'Case Count','Case Count','COMPLETE',@Proc_Step_no,@Proc_Step_name,@RowCount_no);
(@batch_id,@dataflow_name, @package_name,'COMPLETE',@Proc_Step_no,@Proc_Step_name,@RowCount_no);



Expand Down Expand Up @@ -201,8 +201,8 @@ INSERT INTO dbo.[job_flow_log] (
VALUES
(
@batch_id
,'Case Count'
,'Case Count'
,@dataflow_name
,@package_name
,'ERROR'
,@Proc_Step_no
,'ERROR - '+ @Proc_Step_name
Expand Down
1,587 changes: 1,587 additions & 0 deletions db/upgrade/rdb_modern/routines/045-sp_bmird_strep_pneumo_datamart_postprocessing.sql
View file
Open in desktop

Large diffs are not rendered by default.

149 changes: 149 additions & 0 deletions db/upgrade/rdb_modern/tables/061-create-bmird_strep_pneumo_datamart.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,149 @@
--CNDE-2226
IF NOT EXISTS (SELECT 1
FROM sysobjects
WHERE name = 'BMIRD_STREP_PNEUMO_DATAMART'
and xtype = 'U')
BEGIN
CREATE TABLE dbo.BMIRD_STREP_PNEUMO_DATAMART(
INVESTIGATION_KEY BIGINT NOT NULL,
PATIENT_LOCAL_ID VARCHAR(50) NOT NULL,
INVESTIGATION_LOCAL_ID VARCHAR(50) NOT NULL,
DISEASE VARCHAR(50) NULL,
DISEASE_CD VARCHAR(50) NULL,
PATIENT_FIRST_NAME VARCHAR(50) NULL,
PATIENT_LAST_NAME VARCHAR(50) NULL,
PATIENT_DOB DATETIME NULL,
PATIENT_CURRENT_SEX VARCHAR(50) NULL,
AGE_REPORTED NUMERIC NULL,
AGE_REPORTED_UNIT VARCHAR(50) NULL,
PATIENT_STREET_ADDRESS_1 VARCHAR(100) NULL,
PATIENT_STREET_ADDRESS_2 VARCHAR(100) NULL,
PATIENT_CITY VARCHAR(100) NULL,
PATIENT_STATE VARCHAR(100) NULL,
PATIENT_ZIP VARCHAR(20) NULL,
PATIENT_COUNTY VARCHAR(300) NULL,
PATIENT_ETHNICITY VARCHAR(300) NULL,
RACE_CALCULATED VARCHAR(500) NULL,
RACE_CALC_DETAILS VARCHAR(500) NULL,
EARLIEST_RPT_TO_CNTY_DT DATETIME NULL,
EARLIEST_RPT_TO_STATE_DT DATETIME NULL,
HOSPITALIZED VARCHAR(50) NULL,
HOSPITALIZED_ADMISSION_DATE DATETIME NULL,
HOSPITALIZED_DISCHARGE_DATE DATETIME NULL,
HOSPITALIZED_DURATION_DAYS NUMERIC NULL,
HOSPITAL_NAME VARCHAR(100) NULL,
ILLNESS_ONSET_DATE DATETIME NULL,
ILLNESS_END_DATE DATETIME NULL,
DIE_FRM_THIS_ILLNESS_IND VARCHAR(50) NULL,
TYPE_INFECTION_BACTEREMIA VARCHAR(3) NULL,
TYPE_INFECTION_PNEUMONIA VARCHAR(3) NULL,
TYPE_INFECTION_MENINGITIS VARCHAR(3) NULL,
TYPE_INFECTION_EMPYEMA VARCHAR(3) NULL,
TYPE_INFECTION_CELLULITIS VARCHAR(3) NULL,
TYPE_INFECTION_PERITONITIS VARCHAR(3) NULL,
TYPE_INFECTION_PERICARDITIS VARCHAR(3) NULL,
TYPE_INFECTION_PUERPERAL_SEP VARCHAR(3) NULL,
TYPE_INFECTION_SEP_ARTHRITIS VARCHAR(3) NULL,
TYPE_INFECTION_OTHERS_CONCAT VARCHAR(500) NULL,
TYPE_INFECTION_OTHER_SPECIFY VARCHAR(2000) NULL,
BACTERIAL_SPECIES_ISOLATED VARCHAR(50) NULL,
BACTERIAL_SPECIES_ISOLATED_OTH VARCHAR(50) NULL,
FIRST_POSITIVE_CULTURE_DT DATETIME NULL,
STERILE_SITE_BLOOD VARCHAR(3) NULL,
STERILE_SITE_CEREBRAL_SF VARCHAR(3) NULL,
STERILE_SITE_PLEURAL_FLUID VARCHAR(3) NULL,
STERILE_SITE_PERITONEAL_FLUID VARCHAR(3) NULL,
STERILE_SITE_PERICARDIAL_FLUID VARCHAR(3) NULL,
STERILE_SITE_JOINT_FLUID VARCHAR(3) NULL,
STERILE_SITE_OTHERS_CONCAT VARCHAR(500) NULL,
STERILE_SITE_OTHER VARCHAR(2000) NULL,
INTERNAL_BODY_SITE VARCHAR(100) NULL,
NON_STERILE_SITE_1 VARCHAR(50) NULL,
NON_STERILE_SITE_2 VARCHAR(50) NULL,
NON_STERILE_SITE_3 VARCHAR(50) NULL,
NON_STERILE_SITE_OTHER VARCHAR(100) NULL,
UNDERLYING_CONDITION_IND VARCHAR(50) NULL,
UNDERLYING_CONDITION_1 VARCHAR(50) NULL,
UNDERLYING_CONDITION_2 VARCHAR(50) NULL,
UNDERLYING_CONDITION_3 VARCHAR(50) NULL,
UNDERLYING_CONDITION_4 VARCHAR(50) NULL,
UNDERLYING_CONDITION_5 VARCHAR(50) NULL,
UNDERLYING_CONDITION_6 VARCHAR(50) NULL,
UNDERLYING_CONDITION_7 VARCHAR(50) NULL,
UNDERLYING_CONDITION_8 VARCHAR(50) NULL,
OTHER_MALIGNANCY VARCHAR(2000) NULL,
ORGAN_TRANSPLANT VARCHAR(2000) NULL,
OTHER_PRIOR_ILLNESS_1 VARCHAR(100) NULL,
OTHER_PRIOR_ILLNESS_2 VARCHAR(100) NULL,
OTHER_PRIOR_ILLNESS_3 VARCHAR(100) NULL,
CASE_STATUS VARCHAR(50) NULL,
MMWR_WEEK NUMERIC NULL,
MMWR_YEAR NUMERIC NULL,
SAME_PATHOGEN_RECURRENT VARCHAR(50) NULL,
CASE_REPORT_STATUS VARCHAR(50) NULL,
OXACILLIN_INTERPRETATION VARCHAR(50) NULL,
OXACILLIN_ZONE_SIZE NUMERIC NULL,
ANTIMICROBIAL_AGENT_TESTED_1 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_1 VARCHAR(50) NULL,
S_I_R_U_RESULT_1 VARCHAR(50) NULL,
MIC_SIGN_1 VARCHAR(50) NULL,
MIC_VALUE_1 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_2 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_2 VARCHAR(50) NULL,
S_I_R_U_RESULT_2 VARCHAR(50) NULL,
MIC_SIGN_2 VARCHAR(50) NULL,
MIC_VALUE_2 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_3 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_3 VARCHAR(50) NULL,
S_I_R_U_RESULT_3 VARCHAR(50) NULL,
MIC_SIGN_3 VARCHAR(50) NULL,
MIC_VALUE_3 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_4 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_4 VARCHAR(50) NULL,
S_I_R_U_RESULT_4 VARCHAR(50) NULL,
MIC_SIGN_4 VARCHAR(50) NULL,
MIC_VALUE_4 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_5 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_5 VARCHAR(50) NULL,
S_I_R_U_RESULT_5 VARCHAR(50) NULL,
MIC_SIGN_5 VARCHAR(50) NULL,
MIC_VALUE_5 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_6 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_6 VARCHAR(50) NULL,
S_I_R_U_RESULT_6 VARCHAR(50) NULL,
MIC_SIGN_6 VARCHAR(50) NULL,
MIC_VALUE_6 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_7 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_7 VARCHAR(50) NULL,
S_I_R_U_RESULT_7 VARCHAR(50) NULL,
MIC_SIGN_7 VARCHAR(50) NULL,
MIC_VALUE_7 VARCHAR(50) NULL,
ANTIMICROBIAL_AGENT_TESTED_8 VARCHAR(50) NULL,
SUSCEPTABILITY_METHOD_8 VARCHAR(50) NULL,
S_I_R_U_RESULT_8 VARCHAR(50) NULL,
MIC_SIGN_8 VARCHAR(50) NULL,
MIC_VALUE_8 VARCHAR(50) NULL,
ANTIMIC_GT_8_AGENT_AND_RESULT VARCHAR(2000) NULL,
PERSISTENT_DISEASE_IND VARCHAR(50) NULL,
ADD_CULTURE_1_DATE DATETIME NULL,
ADD_CULTURE_1_SITE_1 VARCHAR(50) NULL,
ADD_CULTURE_1_SITE_2 VARCHAR(50) NULL,
ADD_CULTURE_1_SITE_3 VARCHAR(50) NULL,
ADD_CULTURE_1_OTHER_SITE VARCHAR(50) NULL,
ADD_CULTURE_2_DATE DATETIME NULL,
ADD_CULTURE_2_SITE_1 VARCHAR(50) NULL,
ADD_CULTURE_2_SITE_2 VARCHAR(50) NULL,
ADD_CULTURE_2_SITE_3 VARCHAR(50) NULL,
ADD_CULTURE_2_OTHER_SITE VARCHAR(50) NULL,
VACCINE_POLYSACCHARIDE VARCHAR(50) NULL,
VACCINE_CONJUGATE VARCHAR(50) NULL,
PROGRAM_JURISDICTION_OID NUMERIC NULL,
GENERAL_COMMENTS VARCHAR(2000) NULL,
PHC_ADD_TIME DATETIME NULL,
PHC_LAST_CHG_TIME DATETIME NULL,
EVENT_DATE DATETIME NULL,
EVENT_DATE_TYPE VARCHAR(60) NULL,
CULTURE_SEROTYPE VARCHAR(50) NULL,
OTHSEROTYPE VARCHAR(100) NULL
)
END;
9 changes: 9 additions & 0 deletions db/upgrade/rdb_modern/views/013-v_rdb_obs_mapping.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,15 @@
Added CROSS JOIN with two dummy rows. This allows for a code to ALWAYS be represented alongside a null PHC_UID.
This is important for ensuring that when a selection is removed from the UI, it is properly represented and set
to null in the target table.
Reasoning:
If an answer with a certain code is removed from only one investigation,
but that answer exists for another investigation, the changed investigation won't have that attribute set to null.
This is because there wouldn't be a record with NULL for that code in v_rdb_obs_mapping.
Adding the cross join ensures that there is always a record in v_rdb_obs_mapping that is just the information from IMRDBMapping without any observation data.
The process is basically duplicate every row in the IMRDBMapping, join the observation data onto the duplicated rows only (n = 2),
then keep only the records with n=1 (the original rows from IMRDBMapping with, deliberately not containing any observation data) or
records that have a non-null PHC_uid (which would indicate that there are observations for the given code)
the n=1 filter is there to keep only one record for a code in the scenario that no data exists for that code.
*/
CREATE OR ALTER VIEW dbo.v_rdb_obs_mapping
AS
Expand Down
16 changes: 16 additions & 0 deletions liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1159,4 +1159,20 @@ databaseChangeLog:
changes:
- sqlFile:
path: 047-sp_nrt_treatment_postprocessing-001.sql
splitStatements: false
- changeSet:
id: 185
author: liquibase
runOnChange: true
changes:
- sqlFile:
path: 061-create-bmird_strep_pneumo_datamart-001.sql
splitStatements: false
- changeSet:
id: 190
author: liquibase
runOnChange: true
changes:
- sqlFile:
path: 140-sp_bmird_strep_pneumo_datamart_postprocessing-001.sql
splitStatements: false
10 changes: 5 additions & 5 deletions ...ce/src/main/resources/db/rdb_modern/routines/021-sp_nrt_case_count_postprocessing-001.sql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -110,7 +110,7 @@ BEGIN TRY
SELECT @ROWCOUNT_NO = @@ROWCOUNT;
INSERT INTO dbo.[JOB_FLOW_LOG]
(BATCH_ID,[DATAFLOW_NAME],[PACKAGE_NAME] ,[STATUS_TYPE],[STEP_NUMBER],[STEP_NAME],[ROW_COUNT])
VALUES(@BATCH_ID,'CASE_COUNT','CASE_COUNT','START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);
VALUES(@BATCH_ID,@dataflow_name, @package_name,'START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);

SET @PROC_STEP_NO = @PROC_STEP_NO + 1;
SET @PROC_STEP_NAME = ' GENERATING Case Count Table - Insert';
Expand Down Expand Up @@ -159,7 +159,7 @@ BEGIN TRY
SELECT @ROWCOUNT_NO = @@ROWCOUNT;
INSERT INTO dbo.[JOB_FLOW_LOG]
(BATCH_ID,[DATAFLOW_NAME],[PACKAGE_NAME] ,[STATUS_TYPE],[STEP_NUMBER],[STEP_NAME],[ROW_COUNT])
VALUES(@BATCH_ID,'CASE_COUNT','CASE_COUNT','START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);
VALUES(@BATCH_ID,@dataflow_name, @package_name,'START', @PROC_STEP_NO,@PROC_STEP_NAME,@ROWCOUNT_NO);

COMMIT TRANSACTION;

Expand All @@ -171,7 +171,7 @@ BEGIN TRY
INSERT INTO dbo.[job_flow_log]
(batch_id,[Dataflow_Name],[package_Name],[Status_Type] ,[step_number],[step_name],[row_count])
VALUES
(@batch_id,'Case Count','Case Count','COMPLETE',@Proc_Step_no,@Proc_Step_name,@RowCount_no);
(@batch_id,@dataflow_name, @package_name,'COMPLETE',@Proc_Step_no,@Proc_Step_name,@RowCount_no);



Expand Down Expand Up @@ -201,8 +201,8 @@ INSERT INTO dbo.[job_flow_log] (
VALUES
(
@batch_id
,'Case Count'
,'Case Count'
,@dataflow_name
,@package_name
,'ERROR'
,@Proc_Step_no
,'ERROR - '+ @Proc_Step_name
Expand Down
Loading