Added backend of db backup and import system (#207)

Author: Casvt

backend/base/custom_exceptions.py

@@ -9,7 +9,7 @@
 from backend.base.definitions import (ApiResponse, BrokenClientReason,
                                       DownloadService, DownloadType,
                                       EnqueuingDownloadFailureReason,
-                                      KapowarrException)
+                                      InvalidDatabaseReason, KapowarrException)
 from backend.base.logging import LOGGER
 
 
@@ -141,6 +141,52 @@ def api_response(self) -> ApiResponse:
         }
 
 
+class InvalidDatabaseFile(KapowarrException):
+    "The uploaded database file is invalid or not supported"
+
+    def __init__(self, filepath_db: str, reason: InvalidDatabaseReason) -> None:
+        self.filepath_db = filepath_db
+        self.reason = reason
+        LOGGER.warning(
+            "The given database file is invalid: %s (reason=%s)",
+            filepath_db, reason
+        )
+        return
+
+    @property
+    def api_response(self) -> ApiResponse:
+        return {
+            'code': 400,
+            'error': self.__class__.__name__,
+            'result': {
+                'filepath_db': self.filepath_db,
+                'reason': self.reason.value
+            }
+        }
+
+
+class DatabaseFileNotFound(KapowarrException):
+    "The index of the database backup is invalid"
+
+    def __init__(self, backup_index: int) -> None:
+        self.backup_index = backup_index
+        LOGGER.warning(
+            "The given database backup index is invalid: %d",
+            backup_index
+        )
+        return
+
+    @property
+    def api_response(self) -> ApiResponse:
+        return {
+            'code': 400,
+            'error': self.__class__.__name__,
+            'result': {
+                'index': self.backup_index
+            }
+        }
+
+
 # region Rootfolders
 class RootFolderNotFound(KapowarrException):
     "Rootfolder with given ID not found"

backend/base/definitions.py

@@ -50,9 +50,18 @@ class Constants:
     DB_NAME = "Kapowarr.db"
     "Name of database file itself"
 
+    DB_ORIGINAL_NAME = "Kapowarr_original.db"
+    "Name of database file when backed up because a new database is imported"
+
     DB_TIMEOUT = 10.0 # seconds
     "Seconds to wait on database command before timing out"
 
+    DB_REVERT_TIME = 60.0 # seconds
+    """
+    After a new database is imported, how long the user has to access the web-UI
+    before the import is reverted
+    """
+
     DB_MAX_CONCURRENT_CONNECTIONS = 32
     "Maximum allowed database connections to be open at the same time"
 
@@ -289,6 +298,24 @@ class StartType(BaseEnum):
     "A normal restart"
     RESTART_HOSTING_CHANGES = 132
     "A restart because changes to the hosting settings were made"
+    RESTART_DB_CHANGES = 133
+    "A restart because a database import was done"
+
+
+class InvalidDatabaseReason(BaseEnum):
+    "The reason that a database file is invalid"
+
+    DOES_NOT_EXIST = "does_not_exist"
+    "Database file does not exist"
+
+    NOT_KAPOWARR_DB = "not_kapowarr_db"
+    "Uploaded database is not a Kapowarr database file"
+
+    VERSION_NOT_SUPPORTED = "version_not_supported"
+    """
+    Uploaded database is higher version than this Kapowarr installation can\
+    support
+    """
 
 
 class ProxyType(BaseEnum):
@@ -619,6 +646,13 @@ class StatusData(TypedDict):
     """
 
 
+class DatabaseBackupEntry(TypedDict):
+    index: int
+    creation_date: int
+    filepath: str
+    filename: str
+
+
 class FilenameData(TypedDict):
     series: str
     year: Union[int, None]

backend/base/files.py

@@ -762,7 +762,23 @@ def set_volume_folder_owner_group(
 
 
 # region Moving
-def __copy2(src, dst, *, follow_symlinks=True):
+def copy(
+    src,
+    dst,
+    *,
+    follow_symlinks=True
+) -> str:
+    """Copy a file or folder.
+
+    Args:
+        src (str): The source file or folder.
+        dst (str): The destination of the copy.
+        follow_symlinks (bool, optional): Whether to follow symlinks.
+            Defaults to True.
+
+    Returns:
+        str: The destination.
+    """
     try:
         return copy2(src, dst, follow_symlinks=follow_symlinks)
 
@@ -805,12 +821,12 @@ def rename_file(
         # Cannot move folder into itself
         old_before = before
         before = old_before + '_temp'
-        move(old_before, before, copy_function=__copy2)
+        move(old_before, before, copy_function=copy)
 
     create_folder(dirname(after))
 
     # Move file into folder
-    move(before, after, copy_function=__copy2)
+    move(before, after, copy_function=copy)
 
     return
 
@@ -822,7 +838,7 @@ def copy_directory(source: str, target: str) -> None:
         source (str): The current folderpath of the source directory.
         target (str): The desired folderpath to where the directory should be copied.
     """
-    copytree(source, target, copy_function=__copy2)
+    copytree(source, target, copy_function=copy)
     return
 
 

backend/base/helpers.py

@@ -1377,7 +1377,7 @@ def __init__(self, max_processes: Union[int, None] = None) -> None:
         log_filepath = get_log_filepath()
         log_folder = dirname(log_filepath)
         log_file = basename(log_filepath)
-        db_folder = dirname(DBConnection.file)
+        db_folder = dirname(DBConnection.default_file)
         ws_queue = WebSocket().client_manager.queue
 
         super().__init__(

backend/features/tasks.py

@@ -17,6 +17,7 @@
 from backend.implementations.naming import mass_rename
 from backend.implementations.volumes import Volume, refresh_and_scan
 from backend.internals.db import get_db
+from backend.internals.db_backup_import import backup_database
 from backend.internals.server import (Server, TaskAddedEvent, TaskEndedEvent,
                                       TaskStatusEvent, WebSocket)
 
@@ -29,7 +30,8 @@
     # Note: If there are tasks that should be run at the same time,
     #   but per se after each other, put them in that order in the dict.
     'update_all': '0 * * * *', # every hour at minute 0
-    'search_all': '0 0 * * *' # every day at midnight
+    'search_all': '0 0 * * *', # every day at 00:00
+    'backup_db': '0 0 * * 1' # every Monday at 00:00
 }
 
 
@@ -445,7 +447,7 @@ def delete_task_history() -> None:
 class LibraryTask(Task):
     """
     Tasks that inherit from this class signify that they don't work
-    on one specific volume or issue
+    on one specific volume or issue but on all of them
     """
 
 
@@ -868,3 +870,31 @@ def run(self):
                     for result in results
                 ]
         return downloads
+
+
+# region System tasks
+@TaskHandler.register_task("backup_db")
+class BackupDatabase(Task):
+    "Create a backup of the database"
+
+    stop = False
+    message = ''
+    display_title = 'Database Backup'
+
+    @property
+    def volume_id(self) -> None:
+        return None
+
+    @property
+    def issue_id(self) -> None:
+        return None
+
+    def __init__(self) -> None:
+        return
+
+    def run(self):
+        self.message = 'Creating database backup'
+        WebSocket().emit(TaskStatusEvent(self.message))
+
+        backup_database()
+        return

backend/internals/db.py

@@ -102,6 +102,9 @@ class DBConnectionManager(type):
     instances: Dict[int, DBConnection] = {}
 
     def __call__(cls, **kwargs: Any) -> DBConnection:
+        if kwargs.get('db_file'):
+            return super().__call__(**kwargs)
+
         thread_id = current_thread_id()
 
         if (
@@ -126,23 +129,32 @@ def close_connection_of_thread(cls) -> None:
 
 
 class DBConnection(Connection, metaclass=DBConnectionManager):
-    file = ''
+    default_file = ''
 
     def __init__(
         self, *,
+        db_file: Union[str, None] = None,
         timeout: float = Constants.DB_TIMEOUT
     ) -> None:
         """Create a connection with a database
 
         Args:
+            db_file (Union[str, None], optional): The database file to connect
+                to. If `None`, the default file will be used. If something else
+                than the default file is given, then a new connection will
+                always be returned.
+                Defaults to None.
+
             timeout (float, optional): How long to wait before giving up
                 on a command.
                 Defaults to Constants.DB_TIMEOUT.
         """
         self.closed = False
+        self.db_file = db_file or self.default_file
+
         LOGGER.debug(f'Creating connection {self}')
         super().__init__(
-            self.file,
+            self.db_file,
             timeout=timeout,
             detect_types=PARSE_DECLTYPES
         )
@@ -164,20 +176,40 @@ def cursor( # type: ignore
             KapowarrCursor: The database cursor.
         """
         if not hasattr(g, 'cursors'):
-            g.cursors = []
+            g.cursors = {}
+
+        if self.db_file not in g.cursors:
+            g.cursors[self.db_file] = []
 
-        if not g.cursors:
+        if not g.cursors[self.db_file]:
             c = KapowarrCursor(self)
             c.row_factory = Row
-            g.cursors.append(c)
+            g.cursors[self.db_file].append(c)
 
         if not force_new:
-            return g.cursors[0]
+            return g.cursors[self.db_file][0]
         else:
             c = KapowarrCursor(self)
             c.row_factory = Row
-            g.cursors.append(c)
-            return g.cursors[-1]
+            g.cursors[self.db_file].append(c)
+            return g.cursors[self.db_file][-1]
+
+    def create_backup(self, filepath: str) -> None:
+        """Create a backup of the current database.
+
+        Args:
+            filepath (str): What the filepath of the backup will be.
+        """
+        self.execute(
+            "VACUUM INTO ?;",
+            (filepath,)
+        )
+        return
+
+    def merge_wal_files(self) -> None:
+        "Merge the WAL files into the main database file"
+        self.execute("PRAGMA wal_checkpoint(TRUNCATE);")
+        return
 
     def close(self) -> None:
         """Close the database connection"""
@@ -204,6 +236,8 @@ def set_db_location(
     Raises:
         ValueError: Value of `db_folder` exists but is not a folder.
     """
+    from backend.internals.settings import SettingsValues
+
     if db_folder:
         if exists(db_folder) and not isdir(db_folder):
             raise ValueError('Database location is not a folder')
@@ -217,7 +251,8 @@ def set_db_location(
 
     create_folder(dirname(db_file_location))
 
-    DBConnection.file = db_file_location
+    DBConnection.default_file = db_file_location
+    SettingsValues.db_backup_folder = dirname(db_file_location)
 
     return
 
@@ -278,13 +313,14 @@ def close_db(e: Union[BaseException, None] = None) -> None:
 
     try:
         cursors = g.cursors
-        db: DBConnection = cursors[0].connection
-        for c in cursors:
-            c.close()
+        for cursors in g.cursors.values():
+            db: DBConnection = cursors[0].connection
+            for c in cursors:
+                c.close()
+            db.commit()
+            if not current_thread().name.startswith('waitress-'):
+                DBConnectionManager.close_connection_of_thread()
         delattr(g, 'cursors')
-        db.commit()
-        if not current_thread().name.startswith('waitress-'):
-            DBConnectionManager.close_connection_of_thread()
 
     except ProgrammingError:
         pass