add some tests

citusdata · Feb 9, 2024 · a6a684b · a6a684b
1 parent a5748fc
commit a6a684b
Show file tree

Hide file tree

Showing 3 changed files with 240 additions and 0 deletions.
diff --git a/src/test/regress/expected/failure_create_database.out b/src/test/regress/expected/failure_create_database.out
@@ -0,0 +1,184 @@
+SET citus.enable_create_database_propagation TO ON;
+SET client_min_messages TO WARNING;
+SELECT citus.mitmproxy('conn.kill()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE DATABASE db1;
+ERROR:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+ no_temp_databases_on_any_nodes
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+      node_type       |                                                              result
+---------------------------------------------------------------------
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+(2 rows)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE DATABASE db1;
+WARNING:  Commands that are not transaction-safe may result in partial failure, potentially leading to an inconsistent state.
+If the problematic command is a CREATE operation, consider using the 'IF EXISTS' syntax to drop the object,
+if applicable, and then re-attempt the original command.
+ERROR:  canceling statement due to user request
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+ no_temp_databases_on_any_nodes
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+      node_type       |                                                              result
+---------------------------------------------------------------------
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+(2 rows)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE DATABASE db1;
+ERROR:  canceling statement due to user request
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+ no_temp_databases_on_any_nodes
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+      node_type       |                                                              result
+---------------------------------------------------------------------
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+(2 rows)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE DATABASE db1;
+ERROR:  connection to the remote node postgres@localhost:xxxxx failed with the following error: connection not open
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+ no_temp_databases_on_any_nodes
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+      node_type       |                                                              result
+---------------------------------------------------------------------
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+(2 rows)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE DATABASE db1;
+ERROR:  connection not open
+CONTEXT:  while executing command on localhost:xxxxx
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+ no_temp_databases_on_any_nodes
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+      node_type       |                                                              result
+---------------------------------------------------------------------
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": null, "pg_dist_object_record_for_db_exists": false, "stale_pg_dist_object_record_for_a_db_exists": false}
+(2 rows)
+
+SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+CREATE DATABASE db1;
+WARNING:  connection not open
+CONTEXT:  while executing command on localhost:xxxxx
+WARNING:  failed to commit transaction on localhost:xxxxx
+SELECT citus.mitmproxy('conn.allow()');
+ mitmproxy
+---------------------------------------------------------------------
+
+(1 row)
+
+-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time
+SELECT recover_prepared_transactions();
+ recover_prepared_transactions
+---------------------------------------------------------------------
+                             1
+(1 row)
+
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+ no_temp_databases_on_any_nodes
+---------------------------------------------------------------------
+ t
+(1 row)
+
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+      node_type       |                                                                                                                                                                                                                    result
+---------------------------------------------------------------------
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+ worker node (remote) | {"database_properties": {"datacl": null, "datname": "db1", "datctype": "C", "encoding": "UTF8", "datcollate": "C", "tablespace": "pg_default", "daticurules": null, "datallowconn": true, "datconnlimit": -1, "daticulocale": null, "datistemplate": false, "database_owner": "postgres", "datcollversion": null, "datlocprovider": "c"}, "pg_dist_object_record_for_db_exists": true, "stale_pg_dist_object_record_for_a_db_exists": false}
+(2 rows)
+
+DROP DATABASE db1;
+RESET client_min_messages;
diff --git a/src/test/regress/failure_schedule b/src/test/regress/failure_schedule
@@ -35,6 +35,7 @@ test: failure_mx_metadata_sync
 test: failure_mx_metadata_sync_multi_trans
 test: failure_connection_establishment
 test: failure_non_main_db_2pc
+test: failure_create_database
 
 # this test syncs metadata to the workers
 test: failure_failover_to_local_execution

diff --git a/src/test/regress/sql/failure_create_database.sql b/src/test/regress/sql/failure_create_database.sql
@@ -0,0 +1,55 @@
+SET citus.enable_create_database_propagation TO ON;
+SET client_min_messages TO WARNING;
+
+SELECT citus.mitmproxy('conn.kill()');
+CREATE DATABASE db1;
+SELECT citus.mitmproxy('conn.allow()');
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^CREATE DATABASE").cancel(' || pg_backend_pid() || ')');
+CREATE DATABASE db1;
+SELECT citus.mitmproxy('conn.allow()');
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^ALTER DATABASE").cancel(' || pg_backend_pid() || ')');
+CREATE DATABASE db1;
+SELECT citus.mitmproxy('conn.allow()');
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^BEGIN TRANSACTION ISOLATION LEVEL READ COMMITTED").kill()');
+CREATE DATABASE db1;
+SELECT citus.mitmproxy('conn.allow()');
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^PREPARE TRANSACTION").kill()');
+CREATE DATABASE db1;
+SELECT citus.mitmproxy('conn.allow()');
+
+CALL citus_cleanup_orphaned_resources();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+
+SELECT citus.mitmproxy('conn.onQuery(query="^COMMIT PREPARED").kill()');
+CREATE DATABASE db1;
+SELECT citus.mitmproxy('conn.allow()');
+
+-- not call citus_cleanup_orphaned_resources() but recover the prepared transactions this time
+SELECT recover_prepared_transactions();
+SELECT bool_and(result::boolean) AS no_temp_databases_on_any_nodes FROM run_command_on_all_nodes($$SELECT COUNT(*)=0 FROM pg_database WHERE datname LIKE 'citus_temp_database_%'$$);
+SELECT * FROM public.check_database_on_all_nodes($$db1$$) ORDER BY node_type, result;
+
+DROP DATABASE db1;
+
+RESET client_min_messages;