From 52e50d8e279f2ea37b669484fdd6d4600fd9533a Mon Sep 17 00:00:00 2001 From: Ravindran Seshadri <109151953+rseshadri99@users.noreply.github.com> Date: Sat, 8 Mar 2025 22:34:41 -0500 Subject: [PATCH 1/4] Add the dyn datamart procedure for case management data post processing. This procedure migrates the classic etl to rtr framework with event based Ids to be processed and removes any reference to odse Add the dyn datamart procedure for case management data post processing. This procedure migrates the classic etl to rtr framework with event based Ids to be processed and removes any reference to odse --- ...dyn_dm_case_management_post_processing.sql | 209 ++++++++++++++++++ .../db.rdb_modern.changelog-16.1.yaml | 10 +- ...dm_case_management_post_processing-001.sql | 209 ++++++++++++++++++ 3 files changed, 427 insertions(+), 1 deletion(-) create mode 100644 db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql create mode 100644 liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql diff --git a/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql b/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql new file mode 100644 index 00000000..ef2e4303 --- /dev/null +++ b/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql @@ -0,0 +1,209 @@ + + + +CREATE or alter PROCEDURE [dbo].sp_dyn_dm_case_management_post_processing + @batch_id BIGINT, + @DATAMART_NAME VARCHAR(100), + @phc_id_list nvarchar(max) + AS +BEGIN + BEGIN TRY + + -- DECLARE @batch_id BIGINT = 999; DECLARE @DATAMART_NAME VARCHAR(100) = 'TB_LTBI_GA'; + DECLARE @RowCount_no INT ; + DECLARE @Proc_Step_no FLOAT = 0 ; + DECLARE @Proc_Step_Name VARCHAR(200) = '' ; + DECLARE @batch_start_time datetime = null ; + DECLARE @batch_end_time datetime = null ; + DECLARE @nbs_page_form_cd varchar(200)='' + + + + SET @Proc_Step_no = 1; + SET @Proc_Step_Name = 'SP_Start'; + + + + + BEGIN TRANSACTION; + + INSERT INTO dbo.[job_flow_log] ( + batch_id + ,[Dataflow_Name] + ,[package_Name] + ,[Status_Type] + ,[step_number] + ,[step_name] + ,[row_count] + ) + VALUES + ( + @batch_id + ,'DYNAMIC_DATAMART' + ,'RDB.DBO.DynDM_Manage_Case_Management ' + @DATAMART_NAME + ,'START' + ,@Proc_Step_no + ,@Proc_Step_Name + ,0 + ); + + + COMMIT TRANSACTION; + + + BEGIN TRANSACTION; + + SET @Proc_Step_no = @Proc_Step_no + 1; + SET @Proc_Step_Name = 'GENERATING LIST STRING'; + + SET @nbs_page_form_cd = (SELECT FORM_CD FROM dbo.v_nbs_page WHERE DATAMART_NM=@DATAMART_NAME) + + + +/*Creating the list in the rdb_column_name_list with all the user defined column name*/ + +--DECLARE @DATAMART_NAME varchar(100) = 'CONG_SYPHILIS'; + +--DX select * from rdb.dbo.tmp_DynDm_Case_Management_Metadata; + + + + DECLARE @listStr VARCHAR(MAX) = null; +/* +SELECT @listStr = COALESCE(@listStr+',' ,'') + RDB_COLUMN_NM + ' '+ coalesce(USER_DEFINED_COLUMN_NM,'') +FROM rdb.dbo.tmp_DynDm_Case_Management_Metadata with (nolock); +*/ +SELECT rdb_column_nm_list FROM dbo.v_nrt_nbs_d_case_mgmt_rdb_table_metadata where datamart_nm = @DATAMART_NAME; + + + +--DX SELECT @listStr + + + +/*it creates the case_management_data table with the rdb_column_nm associated to the case management plus the investigation key*/ + + -- CREATE TABLE CASE_MANAGEMENT_DATA AS + + SELECT @ROWCOUNT_NO = @@ROWCOUNT; + + INSERT INTO dbo.[job_flow_log] ( batch_id ,[Dataflow_Name] ,[package_Name] ,[Status_Type] ,[step_number] ,[step_name] ,[row_count] ) + VALUES ( @batch_id ,'DYNAMIC_DATAMART' ,'RDB.DBO.DynDM_Manage_Case_Management '+@DATAMART_NAME ,'START' ,@Proc_Step_no , @Proc_Step_Name , @ROWCOUNT_NO ); + + + + COMMIT TRANSACTION; + + + BEGIN TRANSACTION; + + SET @Proc_Step_no = @Proc_Step_no + 1; + SET @Proc_Step_Name = 'GENERATING tmp_DynDm_Case_Management_Data'; + + + + + IF OBJECT_ID('#tmp_DynDm_Case_Management_Data', 'U') IS NOT NULL + drop table #tmp_DynDm_Case_Management_Data; + + + SELECT sd.INVESTIGATION_KEY ,rdb_column_nm_list + INTO #tmp_DynDM_CASE_MANAGEMENT_DATA + FROM #tmp_DynDM_SUMM_DATAMART isd + join dbo.v_nrt_nbs_d_case_mgmt_rdb_table_metadata case_mgmt_meta on case_mgmt_meta.INVESTIGATION_FORM_CD = isd.DISEASE_GRP_CD + and case_mgmt_meta.datamart_nm = @DATAMART_NAME and isd.DISEASE_GRP_CD = @nbs_page_form_cd + and case_mgmt_meta.INVESTIGATION_FORM_CD = @nbs_page_form_cd + INNER JOIN dbo.nrt_investigation_key nrt_inv_key with (nolock) ON isd.investigation_key = nrt_inv_key.d_investigation_key + and nrt_inv_key.case_uid in (SELECT value FROM STRING_SPLIT(@phc_id_list, ',')) + LEFT JOIN dbo.D_CASE_MANAGEMENT case_mgmt ON isd.INVESTIGATION_KEY = case_mgmt.INVESTIGATION_KEY + WHERE case_mgmt.INVESTIGATION_KEY>1 ; + + + SELECT @ROWCOUNT_NO = @@ROWCOUNT; + + INSERT INTO dbo.[job_flow_log] ( batch_id ,[Dataflow_Name] ,[package_Name] ,[Status_Type] ,[step_number] ,[step_name] ,[row_count] ) + VALUES ( @batch_id ,'DYNAMIC_DATAMART' ,'RDB.DBO.DynDM_Manage_Case_Management '+@DATAMART_NAME ,'START' ,@Proc_Step_no , @Proc_Step_Name , @ROWCOUNT_NO ); + + + + COMMIT TRANSACTION; + + + BEGIN TRANSACTION; + + SET @Proc_Step_no = @Proc_Step_no + 1; + SET @Proc_Step_Name = 'SP_COMPLETE'; + + + INSERT INTO dbo.[job_flow_log] ( + batch_id + ,[Dataflow_Name] + ,[package_Name] + ,[Status_Type] + ,[step_number] + ,[step_name] + ,[row_count] + ) + VALUES + ( + @batch_id, + 'DYNAMIC_DATAMART' + ,'RDB.DBO.DynDM_Manage_Case_Management '+@DATAMART_NAME + ,'COMPLETE' + ,@Proc_Step_no + ,@Proc_Step_name + ,0 + ); + + + COMMIT TRANSACTION; + END TRY + + BEGIN CATCH + + + IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION; + + + + DECLARE @ErrorNumber INT = ERROR_NUMBER(); + DECLARE @ErrorLine INT = ERROR_LINE(); + DECLARE @ErrorMessage NVARCHAR(4000) = ERROR_MESSAGE(); + DECLARE @ErrorSeverity INT = ERROR_SEVERITY(); + DECLARE @ErrorState INT = ERROR_STATE(); + + + INSERT INTO dbo.[job_flow_log] ( + batch_id + ,[Dataflow_Name] + ,[package_Name] + ,[Status_Type] + ,[step_number] + ,[step_name] + ,[Error_Description] + ,[row_count] + ) + VALUES + ( + @batch_id + ,'DYNAMIC_DATAMART' + ,'RDB.DBO.DynDM_Manage_Case_Management' + ,'ERROR' + ,@Proc_Step_no + ,'ERROR - '+ @Proc_Step_name + , 'Step -' +CAST(@Proc_Step_no AS VARCHAR(3))+' -' +CAST(@ErrorMessage AS VARCHAR(500)) + ,0 + ); + + + return -1 ; + + END CATCH + +END + +; + + + + diff --git a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml index 736174ee..44f4d41a 100644 --- a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml +++ b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml @@ -1054,4 +1054,12 @@ databaseChangeLog: changes: - sqlFile: path: 060-create-summary-report-case-001.sql - splitStatements: false \ No newline at end of file + splitStatements: false + - changeSet: + id: 151 + author: liquibase + runOnChange: true + changes: + - sqlFile: + path: 130-sp_dyn_dm_case_management_post_processing-001.sql + splitStatements: false diff --git a/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql b/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql new file mode 100644 index 00000000..ef2e4303 --- /dev/null +++ b/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql @@ -0,0 +1,209 @@ + + + +CREATE or alter PROCEDURE [dbo].sp_dyn_dm_case_management_post_processing + @batch_id BIGINT, + @DATAMART_NAME VARCHAR(100), + @phc_id_list nvarchar(max) + AS +BEGIN + BEGIN TRY + + -- DECLARE @batch_id BIGINT = 999; DECLARE @DATAMART_NAME VARCHAR(100) = 'TB_LTBI_GA'; + DECLARE @RowCount_no INT ; + DECLARE @Proc_Step_no FLOAT = 0 ; + DECLARE @Proc_Step_Name VARCHAR(200) = '' ; + DECLARE @batch_start_time datetime = null ; + DECLARE @batch_end_time datetime = null ; + DECLARE @nbs_page_form_cd varchar(200)='' + + + + SET @Proc_Step_no = 1; + SET @Proc_Step_Name = 'SP_Start'; + + + + + BEGIN TRANSACTION; + + INSERT INTO dbo.[job_flow_log] ( + batch_id + ,[Dataflow_Name] + ,[package_Name] + ,[Status_Type] + ,[step_number] + ,[step_name] + ,[row_count] + ) + VALUES + ( + @batch_id + ,'DYNAMIC_DATAMART' + ,'RDB.DBO.DynDM_Manage_Case_Management ' + @DATAMART_NAME + ,'START' + ,@Proc_Step_no + ,@Proc_Step_Name + ,0 + ); + + + COMMIT TRANSACTION; + + + BEGIN TRANSACTION; + + SET @Proc_Step_no = @Proc_Step_no + 1; + SET @Proc_Step_Name = 'GENERATING LIST STRING'; + + SET @nbs_page_form_cd = (SELECT FORM_CD FROM dbo.v_nbs_page WHERE DATAMART_NM=@DATAMART_NAME) + + + +/*Creating the list in the rdb_column_name_list with all the user defined column name*/ + +--DECLARE @DATAMART_NAME varchar(100) = 'CONG_SYPHILIS'; + +--DX select * from rdb.dbo.tmp_DynDm_Case_Management_Metadata; + + + + DECLARE @listStr VARCHAR(MAX) = null; +/* +SELECT @listStr = COALESCE(@listStr+',' ,'') + RDB_COLUMN_NM + ' '+ coalesce(USER_DEFINED_COLUMN_NM,'') +FROM rdb.dbo.tmp_DynDm_Case_Management_Metadata with (nolock); +*/ +SELECT rdb_column_nm_list FROM dbo.v_nrt_nbs_d_case_mgmt_rdb_table_metadata where datamart_nm = @DATAMART_NAME; + + + +--DX SELECT @listStr + + + +/*it creates the case_management_data table with the rdb_column_nm associated to the case management plus the investigation key*/ + + -- CREATE TABLE CASE_MANAGEMENT_DATA AS + + SELECT @ROWCOUNT_NO = @@ROWCOUNT; + + INSERT INTO dbo.[job_flow_log] ( batch_id ,[Dataflow_Name] ,[package_Name] ,[Status_Type] ,[step_number] ,[step_name] ,[row_count] ) + VALUES ( @batch_id ,'DYNAMIC_DATAMART' ,'RDB.DBO.DynDM_Manage_Case_Management '+@DATAMART_NAME ,'START' ,@Proc_Step_no , @Proc_Step_Name , @ROWCOUNT_NO ); + + + + COMMIT TRANSACTION; + + + BEGIN TRANSACTION; + + SET @Proc_Step_no = @Proc_Step_no + 1; + SET @Proc_Step_Name = 'GENERATING tmp_DynDm_Case_Management_Data'; + + + + + IF OBJECT_ID('#tmp_DynDm_Case_Management_Data', 'U') IS NOT NULL + drop table #tmp_DynDm_Case_Management_Data; + + + SELECT sd.INVESTIGATION_KEY ,rdb_column_nm_list + INTO #tmp_DynDM_CASE_MANAGEMENT_DATA + FROM #tmp_DynDM_SUMM_DATAMART isd + join dbo.v_nrt_nbs_d_case_mgmt_rdb_table_metadata case_mgmt_meta on case_mgmt_meta.INVESTIGATION_FORM_CD = isd.DISEASE_GRP_CD + and case_mgmt_meta.datamart_nm = @DATAMART_NAME and isd.DISEASE_GRP_CD = @nbs_page_form_cd + and case_mgmt_meta.INVESTIGATION_FORM_CD = @nbs_page_form_cd + INNER JOIN dbo.nrt_investigation_key nrt_inv_key with (nolock) ON isd.investigation_key = nrt_inv_key.d_investigation_key + and nrt_inv_key.case_uid in (SELECT value FROM STRING_SPLIT(@phc_id_list, ',')) + LEFT JOIN dbo.D_CASE_MANAGEMENT case_mgmt ON isd.INVESTIGATION_KEY = case_mgmt.INVESTIGATION_KEY + WHERE case_mgmt.INVESTIGATION_KEY>1 ; + + + SELECT @ROWCOUNT_NO = @@ROWCOUNT; + + INSERT INTO dbo.[job_flow_log] ( batch_id ,[Dataflow_Name] ,[package_Name] ,[Status_Type] ,[step_number] ,[step_name] ,[row_count] ) + VALUES ( @batch_id ,'DYNAMIC_DATAMART' ,'RDB.DBO.DynDM_Manage_Case_Management '+@DATAMART_NAME ,'START' ,@Proc_Step_no , @Proc_Step_Name , @ROWCOUNT_NO ); + + + + COMMIT TRANSACTION; + + + BEGIN TRANSACTION; + + SET @Proc_Step_no = @Proc_Step_no + 1; + SET @Proc_Step_Name = 'SP_COMPLETE'; + + + INSERT INTO dbo.[job_flow_log] ( + batch_id + ,[Dataflow_Name] + ,[package_Name] + ,[Status_Type] + ,[step_number] + ,[step_name] + ,[row_count] + ) + VALUES + ( + @batch_id, + 'DYNAMIC_DATAMART' + ,'RDB.DBO.DynDM_Manage_Case_Management '+@DATAMART_NAME + ,'COMPLETE' + ,@Proc_Step_no + ,@Proc_Step_name + ,0 + ); + + + COMMIT TRANSACTION; + END TRY + + BEGIN CATCH + + + IF @@TRANCOUNT > 0 ROLLBACK TRANSACTION; + + + + DECLARE @ErrorNumber INT = ERROR_NUMBER(); + DECLARE @ErrorLine INT = ERROR_LINE(); + DECLARE @ErrorMessage NVARCHAR(4000) = ERROR_MESSAGE(); + DECLARE @ErrorSeverity INT = ERROR_SEVERITY(); + DECLARE @ErrorState INT = ERROR_STATE(); + + + INSERT INTO dbo.[job_flow_log] ( + batch_id + ,[Dataflow_Name] + ,[package_Name] + ,[Status_Type] + ,[step_number] + ,[step_name] + ,[Error_Description] + ,[row_count] + ) + VALUES + ( + @batch_id + ,'DYNAMIC_DATAMART' + ,'RDB.DBO.DynDM_Manage_Case_Management' + ,'ERROR' + ,@Proc_Step_no + ,'ERROR - '+ @Proc_Step_name + , 'Step -' +CAST(@Proc_Step_no AS VARCHAR(3))+' -' +CAST(@ErrorMessage AS VARCHAR(500)) + ,0 + ); + + + return -1 ; + + END CATCH + +END + +; + + + + From f417d6035a95d58a636c6f3860f13f78a470ff09 Mon Sep 17 00:00:00 2001 From: Ravindran Seshadri <109151953+rseshadri99@users.noreply.github.com> Date: Sat, 8 Mar 2025 23:47:35 -0500 Subject: [PATCH 2/4] Update db.rdb_modern.changelog-16.1.yaml update liquibase changelog --- .../resources/db/changelog/db.rdb_modern.changelog-16.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml index 44f4d41a..c8bc7028 100644 --- a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml +++ b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml @@ -1056,7 +1056,7 @@ databaseChangeLog: path: 060-create-summary-report-case-001.sql splitStatements: false - changeSet: - id: 151 + id: 155 author: liquibase runOnChange: true changes: From 3cc047fdbbcb6f0e119ed0da1b26061a3b3c7151 Mon Sep 17 00:00:00 2001 From: Ravindran Seshadri <109151953+rseshadri99@users.noreply.github.com> Date: Sun, 9 Mar 2025 01:04:48 -0500 Subject: [PATCH 3/4] Update db.rdb_modern.changelog-16.1.yaml liquibase change log update --- .../resources/db/changelog/db.rdb_modern.changelog-16.1.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml index c8bc7028..788c3d01 100644 --- a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml +++ b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml @@ -1056,7 +1056,7 @@ databaseChangeLog: path: 060-create-summary-report-case-001.sql splitStatements: false - changeSet: - id: 155 + id: 160 author: liquibase runOnChange: true changes: From 9d877577995a0e77cfe4cac7a3ecc5bc00a1cd3c Mon Sep 17 00:00:00 2001 From: Ravindran Seshadri <109151953+rseshadri99@users.noreply.github.com> Date: Mon, 10 Mar 2025 23:50:22 -0400 Subject: [PATCH 4/4] Proc name change Proc name change --- ...ing.sql => 130-sp_dyn_dm_case_management_postprocessing.sql} | 2 +- .../resources/db/changelog/db.rdb_modern.changelog-16.1.yaml | 2 +- ...sql => 130-sp_dyn_dm_case_management_postprocessing-001.sql} | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) rename db/upgrade/rdb_modern/routines/{130-sp_dyn_dm_case_management_post_processing.sql => 130-sp_dyn_dm_case_management_postprocessing.sql} (98%) rename liquibase-service/src/main/resources/db/rdb_modern/routines/{130-sp_dyn_dm_case_management_post_processing-001.sql => 130-sp_dyn_dm_case_management_postprocessing-001.sql} (98%) diff --git a/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql b/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_postprocessing.sql similarity index 98% rename from db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql rename to db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_postprocessing.sql index ef2e4303..c5059305 100644 --- a/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing.sql +++ b/db/upgrade/rdb_modern/routines/130-sp_dyn_dm_case_management_postprocessing.sql @@ -1,7 +1,7 @@ -CREATE or alter PROCEDURE [dbo].sp_dyn_dm_case_management_post_processing +CREATE or alter PROCEDURE [dbo].sp_dyn_dm_case_management_postprocessing @batch_id BIGINT, @DATAMART_NAME VARCHAR(100), @phc_id_list nvarchar(max) diff --git a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml index 788c3d01..3bda9d6d 100644 --- a/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml +++ b/liquibase-service/src/main/resources/db/changelog/db.rdb_modern.changelog-16.1.yaml @@ -1061,5 +1061,5 @@ databaseChangeLog: runOnChange: true changes: - sqlFile: - path: 130-sp_dyn_dm_case_management_post_processing-001.sql + path: 130-sp_dyn_dm_case_management_postprocessing-001.sql splitStatements: false diff --git a/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql b/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_postprocessing-001.sql similarity index 98% rename from liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql rename to liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_postprocessing-001.sql index ef2e4303..c5059305 100644 --- a/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_post_processing-001.sql +++ b/liquibase-service/src/main/resources/db/rdb_modern/routines/130-sp_dyn_dm_case_management_postprocessing-001.sql @@ -1,7 +1,7 @@ -CREATE or alter PROCEDURE [dbo].sp_dyn_dm_case_management_post_processing +CREATE or alter PROCEDURE [dbo].sp_dyn_dm_case_management_postprocessing @batch_id BIGINT, @DATAMART_NAME VARCHAR(100), @phc_id_list nvarchar(max)