feat: log runs jsonl before db removal

.github/workflows/tests.yml

@@ -101,6 +101,7 @@ jobs:
           bash scripts/install.sh
 
       - name: Initialize Browser library
+        timeout-minutes: 5
         run: |
           rfbrowser init
 

docs/basic-command-line-interface-cli.md

@@ -116,6 +116,11 @@ robotdashboard -r alias=some_cool_alias,tag=prod,tag=dev -r alias=alias12345
 robotdashboard -r limit=10
 robotdashboard -r age=10d # (y)ear/(d)ay/(h)our/(m)inute/(s)econd supported
 robotdashboard -r age=-10d
+# Log data of removed runs in jsonl  
+robotdashboard -r limit=10 --logremoved "/myLogDir/removedRuns.jsonl"
+robotdashboard -r limit=10 --logremoved "run:suite:/myLogDir/removedRuns.jsonl"
+robotdashboard -r limit=10 --logremoved all
+robotdashboard -r limit=10 --logremoved run:keyword
 ```
 - Optional: `-r` or `--removeruns` specifies one or more runs to remove.  
 - Multiple values are separated by commas (,).  
@@ -125,6 +130,11 @@ robotdashboard -r age=-10d
 - With limit=10 only the 10 most recent runs will be kept, all others will be removed.  
 - With age=10d only runs _**older**_ than 10 days will be removed  
 - With age=-10d only runs _**younger**_ than 10 days will be removed  
+- Optional: `--logremoved` logs run data to a `.jsonl` file before removal.  
+- Format: `[types:]path` where types are colon-separated from `run`, `suite`, `test`, `keyword`, `all`.  
+- If no types are specified, defaults to `all` (runs, suites, tests and keywords).  
+- If no path is provided, defaults to `robot_removed_runs.jsonl` in the current directory.  
+- Each removed run is appended as one JSON line containing the selected data types.  
 
 ### Use a custom database class
 ```bash

requirements-test.txt

@@ -1,5 +1,5 @@
-robotframework==7.4.1
-robotframework-pabot==5.2.0
-robotframework-browser==19.12.4
+robotframework==7.4.2
+robotframework-pabot==5.2.2
+robotframework-browser==20.0.0
 robotframework-databaselibrary==2.4.1
-robotframework-doctestlibrary==0.31.0
+robotframework-doctestlibrary==0.33.0

robotframework_dashboard/arguments.py

@@ -4,8 +4,16 @@
 from re import split
 from os import getcwd
 from os.path import join, exists
+from typing import NamedTuple
 from .version import __version__
 
+_VALID_LOG_TYPES = {"run", "suite", "test", "keyword", "all"}
+
+
+class LogRemovedConfig(NamedTuple):
+    types: list
+    path: str
+
 
 class dotdict(dict):
     """dot.notation access to dictionary attributes"""
@@ -275,6 +283,24 @@ def _parse_arguments(self):
             nargs="*",
             default=None,
         )
+        db_group.add_argument(
+            "--logremoved",
+            metavar="TYPES_PATH",
+            help=(
+                "Log (append if file exists) run data to jsonl before removing from the db\n"
+                "  • Supply what types to log (default=all) and (optional) a path to the desired .jsonl\n"
+                "Examples:\n"
+                "  • '--logremoved \"/tmp/removed_runs.jsonl\"'\n"
+                "  • '--logremoved \"run:suite\"'\n"
+                "  • '--logremoved \"all:/tmp/removed_runs.jsonl\"'\n"
+            ),
+            action="store",
+            type=str,
+            nargs="?",
+            const="all",
+            default=None,
+            dest="logremoved",
+        )
         db_group.add_argument(
             "-c",
             "--databaseclass",
@@ -605,6 +631,23 @@ def _process_arguments(self, arguments):
         ssl_certfile = arguments.ssl_certfile
         ssl_keyfile = arguments.ssl_keyfile
 
+        # handles --logremoved: "type1:type2:path" or "path" or "type1:type2"
+        log_removed = None
+        if arguments.logremoved:
+            parts = arguments.logremoved.split(":")
+            types, remaining = [], list(parts)
+            # grab element types, leave path, for windows drive letter X: parse
+            for part in parts:
+                if part in _VALID_LOG_TYPES:
+                    types.append(part)
+                    remaining.pop(0)
+                else:
+                    break
+            if not types:
+                types = ["all"]
+            path = ":".join(remaining) if remaining else "robot_removed_runs.jsonl"
+            log_removed = LogRemovedConfig(types=types, path=path)
+
         # validates argument combinations
         self._check_argument_errors(arguments, outputs, outputfolderpaths, force_json_config, database_class)
         self._check_argument_warnings(arguments, outputs, outputfolderpaths, use_logs, generate_dashboard, no_autoupdate, offline_dependencies)
@@ -640,5 +683,6 @@ def _process_arguments(self, arguments):
             "ssl_certfile": ssl_certfile,
             "ssl_keyfile": ssl_keyfile,
             "log_url": arguments.logurl,
+            "log_removed": log_removed,
         }
         return dotdict(provided_args)

robotframework_dashboard/database.py

@@ -6,6 +6,7 @@
 from time import time
 from datetime import datetime, timezone, timedelta
 from typing import Union
+from json import dumps
 
 # Explicit adapter for datetime -> ISO string, replacing the deprecated default
 # behaviour removed in Python 3.12+. Compatible with Python 3.8+.
@@ -14,14 +15,16 @@
 
 
 class DatabaseProcessor(AbstractDatabaseProcessor):
-    def __init__(self, database_path: Path):
+    def __init__(self, database_path: Path, log_removed=None):
         """This function should handle the connection to the database
         And if required the creation of the tables"""
         self.database_path = database_path
         # handle possible subdirectories before creating database with sqlite
         path = Path(self.database_path)
         path.parent.mkdir(exist_ok=True, parents=True)
         self.connection: sqlite3.Connection
+        self.log_removed_path = log_removed.path if log_removed else None
+        self.log_removed_types = log_removed.types if log_removed else []
         # create tables if required
         self.open_database()
         self._create_tables()
@@ -384,6 +387,40 @@ def _get_run_paths(self):
             run_paths[entry["run_start"]] = entry.get("path") or ""
         return run_paths
 
+    def _get_run_data(self, run_start):
+        cursor = self.connection.cursor()
+        cursor.execute(GET_RUN_INFO_BY_RUN_START, (run_start,))
+        row = cursor.fetchone()
+        if not row:
+            return None
+        columns = [col[0] for col in cursor.description]
+        return dict(zip(columns, row))
+
+    def _get_rows_for_run_start(self, table, run_start):
+        cursor = self.connection.cursor()
+        cursor.execute(f"SELECT * FROM {table} WHERE run_start = ?", (run_start,))
+        rows = cursor.fetchall()
+        columns = [col[0] for col in cursor.description]
+        return [dict(zip(columns, row)) for row in rows]
+
+    def _collect_log_entry(self, run_start):
+        entry = {}
+        types = self.log_removed_types
+        include_all = "all" in types
+        if include_all or "run" in types:
+            entry["run"] = self._get_run_data(run_start)
+        if include_all or "suite" in types:
+            entry["suites"] = self._get_rows_for_run_start("suites", run_start)
+        if include_all or "test" in types:
+            entry["tests"] = self._get_rows_for_run_start("tests", run_start)
+        if include_all or "keyword" in types:
+            entry["keywords"] = self._get_rows_for_run_start("keywords", run_start)
+        return entry
+
+    def _log_run_jsonl(self, logpath, entry):
+        with Path(logpath).open("a", encoding="utf-8") as f:
+            _ = f.write(dumps(entry, default=str) + "\n")
+
     def remove_runs(self, remove_runs: list):
         """This function removes all provided runs and all their corresponding data"""
         run_starts, run_names, run_aliases, run_tags, _ = self._get_runs()
@@ -526,13 +563,18 @@ def _remove_by_age(self, run_query: str, run_starts: list):
 
     def _remove_run(self, run_start: str):
         """Helper function to remove the data from all tables"""
-        self.connection.cursor().execute(DELETE_FROM_RUNS.format(run_start=run_start))
-        self.connection.cursor().execute(DELETE_FROM_SUITES.format(run_start=run_start))
-        self.connection.cursor().execute(DELETE_FROM_TESTS.format(run_start=run_start))
-        self.connection.cursor().execute(
-            DELETE_FROM_KEYWORDS.format(run_start=run_start)
-        )
-        self.connection.commit()
+        entry = self._collect_log_entry(run_start) if self.log_removed_path else None
+        with self.connection:
+            cursor = self.connection.cursor()
+            cursor.execute(DELETE_FROM_RUNS.format(run_start=run_start))
+            if cursor.rowcount > 0:
+                cursor.execute(DELETE_FROM_SUITES.format(run_start=run_start))
+                cursor.execute(DELETE_FROM_TESTS.format(run_start=run_start))
+                cursor.execute(DELETE_FROM_KEYWORDS.format(run_start=run_start))
+                # Log inside the transaction: if the write fails, the transaction
+                # rolls back and the run is not deleted.
+                if self.log_removed_path and entry:
+                    self._log_run_jsonl(self.log_removed_path, entry)
 
     def vacuum_database(self):
         """This function vacuums the database to reduce the size after removing runs"""

robotframework_dashboard/main.py

@@ -40,6 +40,7 @@ def main():
         arguments.timezone,
         arguments.log_url,
         arguments.custom_filters,
+        arguments.log_removed,
     )
     # If arguments.start_server is provided override some required args
     if arguments.start_server:

robotframework_dashboard/queries.py

@@ -46,3 +46,5 @@
 UPDATE_RUN_PATH = """ UPDATE runs SET path="{path}" WHERE run_start="{run_start}" """
 
 VACUUM_DATABASE = """ VACUUM """
+
+GET_RUN_INFO_BY_RUN_START = """ SELECT * FROM runs WHERE run_start = ? """

robotframework_dashboard/robotdashboard.py

@@ -7,6 +7,7 @@
 from pathlib import Path
 from datetime import datetime
 from typing import Optional
+from .arguments import LogRemovedConfig
 
 class RobotDashboard:
     """Class that provides all functionality that robotdashboard has to offer
@@ -34,6 +35,7 @@ def __init__(
         timezone: str = "",
         log_url: Optional[str] = None,
         custom_filters: str = "",
+        log_removed: Optional[LogRemovedConfig] = None,
     ):
         """Sets the parameters provided in the command line"""
         self.database_path = database_path
@@ -57,16 +59,23 @@ def __init__(
         self.timezone = timezone
         self.log_url = log_url
         self.custom_filters = custom_filters
+        self.log_removed = log_removed
 
     def initialize_database(self, suppress=True):
         """Function that initializes the database if it does not exist
         Also makes a connection that is used internally in the RobotDashboard class functions
         """
         console = ""
+        if self.log_removed and self.log_removed.path:
+            log_parent = Path(self.log_removed.path).parent
+            if not log_parent.exists():
+                message = f"  ERROR: Directory for '--logremoved' does not exist: '{log_parent}'"
+                console += self._print_console(message)
+                raise FileNotFoundError(message)
         if not suppress:
             console += self._print_console(f" 1. Database preparation")
         if not self.database_class:
-            self.database = DatabaseProcessor(self.database_path)
+            self.database = DatabaseProcessor(self.database_path, self.log_removed)
         else:
             if not suppress:
                 console += self._print_console(
@@ -80,6 +89,14 @@ def initialize_database(self, suppress=True):
             module = importlib.util.module_from_spec(spec)
             spec.loader.exec_module(module)
             self.database = module.DatabaseProcessor(self.database_path)
+            is_rm_log_unsupported = not hasattr(self.database, "log_removed_path")
+            self.database.log_removed_path = self.log_removed.path if self.log_removed else None
+            self.database.log_removed_types = self.log_removed.types if self.log_removed else []
+            if is_rm_log_unsupported and self.log_removed and not suppress:
+                console += self._print_console(
+                    "  WARNING The custom database class does not explicitly support '--logremoved'. "
+                    "Logging might be skipped if the custom class overrides the deletion logic."
+                )
         if not suppress:
             console += self._print_console(
                 f"  created database: '{self.database_path}'"

tests/python/test_arguments.py

@@ -140,6 +140,7 @@ def _make_namespace(**kwargs):
         "ssl_keyfile": None,
         "logurl": None,
         "custom_filters": None,
+        "logremoved": None,
     }
     defaults.update(kwargs)
     return argparse.Namespace(**defaults)

tests/python/test_database.py

@@ -4,6 +4,7 @@
 import pytest
 from robotframework_dashboard.database import DatabaseProcessor
 from robotframework_dashboard.processors import OutputProcessor
+from robotframework_dashboard.arguments import LogRemovedConfig
 
 OUTPUTS_DIR = Path(__file__).parent.parent / "robot" / "resources" / "outputs"
 SAMPLE_XML = OUTPUTS_DIR / "output-20250313-002134.xml"
@@ -506,6 +507,86 @@ def test_get_data_null_test_tags_and_id(db):
     assert data["tests"][0]["tags"] == ""
 
 
+# --- _get_run_data ---
+def test_get_run_data_returns_dict_for_existing_run(populated_db):
+    populated_db.open_database()
+    run_start = populated_db.get_data()["runs"][0]["run_start"]
+    result = populated_db._get_run_data(run_start)
+    populated_db.close_database()
+    assert result is not None
+    assert result["run_start"] == run_start
+
+
+def test_get_run_data_returns_none_for_missing_run(populated_db):
+    populated_db.open_database()
+    result = populated_db._get_run_data("2000-01-01 00:00:00.000000")
+    populated_db.close_database()
+    assert result is None
+
+
+# --- _log_run_jsonl ---
+
+def test_log_run_jsonl_creates_file_and_writes_entry(tmp_path):
+    import json
+    db = DatabaseProcessor(tmp_path / "test.db")
+    logpath = tmp_path / "removed.jsonl"
+    db._log_run_jsonl(logpath, {"run_start": "2025-01-01", "name": "My Run"})
+    parsed = json.loads(logpath.read_text())
+    assert parsed["run_start"] == "2025-01-01"
+
+
+def test_log_run_jsonl_appends_on_successive_calls(tmp_path):
+    db = DatabaseProcessor(tmp_path / "test.db")
+    logpath = tmp_path / "removed.jsonl"
+    db._log_run_jsonl(logpath, {"run_start": "2025-01-01"})
+    db._log_run_jsonl(logpath, {"run_start": "2025-01-02"})
+    assert len(logpath.read_text().splitlines()) == 2
+
+
+def test_log_run_jsonl_serializes_datetime(tmp_path):
+    import json
+    from datetime import datetime
+    db = DatabaseProcessor(tmp_path / "test.db")
+    logpath = tmp_path / "removed.jsonl"
+    db._log_run_jsonl(logpath, {"run_start": datetime(2025, 1, 1, 12, 0, 0)})
+    parsed = json.loads(logpath.read_text())
+    assert "2025-01-01" in parsed["run_start"]
+
+
+# --- _remove_run with logging ---
+
+def test_remove_run_with_logging_writes_jsonl_and_deletes(tmp_path):
+    import json
+    logpath = tmp_path / "removed.jsonl"
+    db = DatabaseProcessor(tmp_path / "test.db", log_removed=LogRemovedConfig(types=["all"], path=str(logpath)))
+    processor = OutputProcessor(SAMPLE_XML)
+    processor.get_run_start()
+    db.open_database()
+    db.insert_output_data(processor.get_output_data(), [], "alias", SAMPLE_XML, None, timezone="+00:00")
+    run_start = db.get_data()["runs"][0]["run_start"]
+    db.remove_runs([f"run_start={run_start}"])
+    assert len(db.get_data()["runs"]) == 0
+    db.close_database()
+    lines = logpath.read_text().splitlines()
+    assert len(lines) == 1
+    assert json.loads(lines[0])["run"]["run_alias"] == "alias"
+
+
+def test_remove_run_logging_failure_rolls_back_delete(tmp_path):
+    from unittest.mock import patch
+    logpath = tmp_path / "removed.jsonl"
+    db = DatabaseProcessor(tmp_path / "test.db", log_removed=LogRemovedConfig(types=["all"], path=str(logpath)))
+    processor = OutputProcessor(SAMPLE_XML)
+    processor.get_run_start()
+    db.open_database()
+    db.insert_output_data(processor.get_output_data(), [], "alias", SAMPLE_XML, None, timezone="+00:00")
+    run_start = db.get_data()["runs"][0]["run_start"]
+    with patch.object(db, "_log_run_jsonl", side_effect=OSError("disk full")):
+        db.remove_runs([f"run_start={run_start}"])
+    assert len(db.get_data()["runs"]) == 1
+    db.close_database()
+
+
 # --- remove_runs bare except branch ---
 
 def test_remove_runs_exception_branch_logs_error(populated_db):