[Issue #313] add test incr_restore.IncrRestoreTest.test_incr_restore_issue_313

Author: kulaginm

src/archive.c

@@ -148,7 +148,7 @@ do_archive_push(InstanceState *instanceState, InstanceConfig *instance, char *wa
 		elog(ERROR, "getcwd() error");
 
 	/* verify that archive-push --instance parameter is valid */
-	system_id = get_system_identifier(current_dir);
+	system_id = get_system_identifier(current_dir, XLOG_CONTROL_FILE, FIO_DB_HOST);
 
 	if (instance->pgdata == NULL)
 		elog(ERROR, "Cannot read pg_probackup.conf for this instance");

src/backup.c

@@ -983,7 +983,7 @@ check_system_identifiers(PGconn *conn, char *pgdata)
 	uint64		system_id_conn;
 	uint64		system_id_pgdata;
 
-	system_id_pgdata = get_system_identifier(pgdata);
+	system_id_pgdata = get_system_identifier(pgdata, XLOG_CONTROL_FILE, FIO_DB_HOST);
 	system_id_conn = get_remote_system_identifier(conn);
 
 	/* for checkdb check only system_id_pgdata and system_id_conn */

src/init.c

@@ -57,7 +57,7 @@ do_add_instance(InstanceState *instanceState, InstanceConfig *instance)
 						 "(-D, --pgdata)");
 
 	/* Read system_identifier from PGDATA */
-	instance->system_identifier = get_system_identifier(instance->pgdata);
+	instance->system_identifier = get_system_identifier(instance->pgdata, XLOG_CONTROL_FILE, FIO_DB_HOST);
 	/* Starting from PostgreSQL 11 read WAL segment size from PGDATA */
 	instance->xlog_seg_size = get_xlog_seg_size(instance->pgdata);
 

src/pg_probackup.h

@@ -1139,7 +1139,7 @@ extern XLogRecPtr get_next_record_lsn(const char *archivedir, XLogSegNo	segno, T
 extern TimeLineID get_current_timeline(PGconn *conn);
 extern TimeLineID get_current_timeline_from_control(bool safe);
 extern XLogRecPtr get_checkpoint_location(PGconn *conn);
-extern uint64 get_system_identifier(const char *pgdata_path);
+extern uint64 get_system_identifier(const char *pgdata_path, const char *pg_control_filename, fio_location location);
 extern uint64 get_remote_system_identifier(PGconn *conn);
 extern uint32 get_data_checksum_version(bool safe);
 extern pg_crc32c get_pgcontrol_checksum(const char *pgdata_path);

src/restore.c

@@ -2233,7 +2233,7 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
 	bool    postmaster_is_up = false;
 	bool    backup_label_exists = false;
 	pid_t   pid;
-	char    backup_label[MAXPGPATH];
+	char    filename[MAXPGPATH];
 
 	/* check postmaster pid */
 	pid = fio_check_postmaster(pgdata, FIO_DB_HOST);
@@ -2268,7 +2268,12 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
 	 */
 	elog(INFO, "Trying to read pg_control file in destination direstory");
 
-	system_id_pgdata = get_system_identifier(pgdata);
+	/* [Issue #313] check for previous failed incremental restore */
+	join_path_components(filename, instance_config.pgdata, XLOG_CONTROL_BAK_FILE);
+	if (fio_access(filename, F_OK, FIO_DB_HOST) == 0)
+		system_id_pgdata = get_system_identifier(pgdata, XLOG_CONTROL_BAK_FILE, FIO_DB_HOST);
+	else
+		system_id_pgdata = get_system_identifier(pgdata, XLOG_CONTROL_FILE, FIO_DB_HOST);
 
 	if (system_id_pgdata == instance_config.system_identifier)
 		system_id_match = true;
@@ -2283,8 +2288,8 @@ check_incremental_compatibility(const char *pgdata, uint64 system_identifier,
 	 */
 	if (incremental_mode == INCR_LSN)
 	{
-		join_path_components(backup_label, pgdata, "backup_label");
-		if (fio_access(backup_label, F_OK, FIO_DB_HOST) == 0)
+		join_path_components(filename, pgdata, PG_BACKUP_LABEL_FILE);
+		if (fio_access(filename, F_OK, FIO_DB_HOST) == 0)
 		{
 			elog(WARNING, "Destination directory contains \"backup_control\" file. "
 				"This does NOT mean that you should delete this file and retry, only that "

src/util.c

@@ -249,14 +249,14 @@ get_checkpoint_location(PGconn *conn)
 }
 
 uint64
-get_system_identifier(const char *pgdata_path)
+get_system_identifier(const char *pgdata_path, const char *pg_control_filename, fio_location location)
 {
 	ControlFileData ControlFile;
 	char	   *buffer;
 	size_t		size;
 
 	/* First fetch file... */
-	buffer = slurpFile(pgdata_path, XLOG_CONTROL_FILE, &size, false, FIO_DB_HOST);
+	buffer = slurpFile(pgdata_path, pg_control_filename, &size, false, location);
 	if (buffer == NULL)
 		return 0;
 	digestControlFile(&ControlFile, buffer, size);

tests/helpers/ptrack_helpers.py

@@ -1602,11 +1602,11 @@ def pgdata_content(self, pgdata, ignore_ptrack=True, exclude_dirs=None):
         files_to_ignore = [
             'postmaster.pid', 'postmaster.opts',
             'pg_internal.init', 'postgresql.auto.conf',
-            'backup_label', 'tablespace_map', 'recovery.conf',
-            'ptrack_control', 'ptrack_init', 'pg_control',
-            'probackup_recovery.conf', 'recovery.signal',
-            'standby.signal', 'ptrack.map', 'ptrack.map.mmap',
-            'ptrack.map.tmp'
+            'backup_label', 'backup_label.old', 'tablespace_map',
+            'recovery.conf', 'recovery.done', 'ptrack_control',
+            'ptrack_init', 'pg_control', 'probackup_recovery.conf',
+            'recovery.signal', 'standby.signal',
+            'ptrack.map', 'ptrack.map.mmap', 'ptrack.map.tmp'
         ]
 
         if exclude_dirs:

tests/incr_restore.py

@@ -9,7 +9,7 @@
 import hashlib
 import shutil
 import json
-from testgres import QueryException
+from testgres import QueryException, StartNodeException
 
 
 module_name = 'incr_restore'
@@ -2436,3 +2436,82 @@ def test_incremental_pg_filenode_map(self):
         self.del_test_dir(module_name, fname)
 
 # check that MinRecPoint and BackupStartLsn are correctly used in case of --incrementa-lsn
+
+
+    # @unittest.skip("skip")
+    def test_incr_restore_issue_313(self):
+        """
+        Check that failed incremental restore can be restarted
+        """
+        fname = self.id().split('.')[3]
+
+        node = self.make_simple_node(
+            base_dir = os.path.join(module_name, fname, 'node'),
+            set_replication = True,
+            initdb_params = ['--data-checksums'])
+
+        backup_dir = os.path.join(self.tmp_path, module_name, fname, 'backup')
+        self.init_pb(backup_dir)
+        self.add_instance(backup_dir, 'node', node)
+        self.set_archiving(backup_dir, 'node', node)
+        node.slow_start()
+
+        node.pgbench_init(scale = 50)
+
+        full_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='full')
+
+        pgbench = node.pgbench(
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
+            options=['-T', '10', '-c', '1', '--no-vacuum'])
+        pgbench.wait()
+        pgbench.stdout.close()
+
+        last_backup_id = self.backup_node(backup_dir, 'node', node, backup_type='delta')
+
+        pgdata = self.pgdata_content(node.data_dir)
+        node.cleanup()
+
+        self.restore_node(backup_dir, 'node', node, backup_id=full_backup_id)
+
+        count = 0
+        filelist = self.get_backup_filelist(backup_dir, 'node', last_backup_id)
+        for file in filelist:
+            # count only nondata files
+            if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0:
+                count += 1
+
+        gdb = self.restore_node(backup_dir, 'node', node, gdb=True,
+            backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum'])
+        gdb.verbose = False
+        gdb.set_breakpoint('restore_non_data_file')
+        gdb.run_until_break()
+        gdb.continue_execution_until_break(count - 2)
+        gdb.quit()
+
+        try:
+            node.slow_start()
+            # we should die here because exception is what we expect to happen
+            self.assertEqual(
+                1, 0,
+                "Expecting Error because backup is not fully restored")
+        except StartNodeException as e:
+            self.assertIn(
+                'Cannot start node',
+                e.message,
+                '\n Unexpected Error Message: {0}\n CMD: {1}'.format(
+                    repr(e.message), self.cmd))
+
+        with open(os.path.join(node.logs_dir, 'postgresql.log'), 'r') as f:
+            self.assertIn(
+                "postgres: could not find the database system",
+                f.read())
+
+        self.restore_node(backup_dir, 'node', node,
+            backup_id=last_backup_id, options=['--progress', '--incremental-mode=checksum'])
+        node.slow_start()
+
+        self.compare_pgdata(pgdata, self.pgdata_content(node.data_dir))
+
+        # Clean after yourself
+        node.stop()
+        self.del_test_dir(module_name, fname)

tests/restore.py

@@ -3880,7 +3880,7 @@ def test_restore_issue_313(self):
         node.cleanup()
 
         count = 0
-        filelist =  self.get_backup_filelist(backup_dir, 'node', backup_id)
+        filelist = self.get_backup_filelist(backup_dir, 'node', backup_id)
         for file in filelist:
             # count only nondata files
             if int(filelist[file]['is_datafile']) == 0 and int(filelist[file]['size']) > 0: