feat(logging): add log rotation and transition default server posture to stdout/stderr

Introduce thread-safe log rotation via RotatingFileSink and transition direct lemond executions to console-first logging with optional rotating file logging.

- Implement RotatingFileSink with size-triggered rotation, max_files backup retention, excess backup pruning, backup slot fallback, and in-place truncation for max_files=0.
- Add configuration settings log_max_file_size_mb (1..2048) and log_max_files (0..100) with 64-bit validation across CLI, RuntimeConfig, and config.json.
- Support --log-file, --log-max-size-mb, and --log-max-files CLI overrides as in-memory runtime parameters without mutating persistent config.
- Add comprehensive C++ unit test suite (test_log_rotation) and Python integration suite (test/server_log_rotation.py).

Author: abn

.github/actions/capture-server-logs/action.yml

@@ -13,21 +13,18 @@ runs:
       run: |
         New-Item -ItemType Directory -Path server-logs -Force | Out-Null
 
-        # Check common log locations
-        $candidates = @(
-          "$env:TEMP\lemonade-server.log",
-          "$env:TEMP\lemond.log",
-          "$env:LOCALAPPDATA\lemonade_server\lemonade-server.log"
-        )
-
+        # Check common log locations (including rotated log files .1, .2, etc.)
+        $logFiles = Get-ChildItem -Path "$env:TEMP", "$env:LOCALAPPDATA\lemonade_server" -Filter "lemonade-server.log*" -ErrorAction SilentlyContinue
         $found = $false
-        foreach ($logFile in $candidates) {
-          if (Test-Path $logFile) {
-            Write-Host "=== Last 200 lines of $logFile ==="
-            Get-Content $logFile -Tail 200
-            Copy-Item $logFile server-logs/ -ErrorAction SilentlyContinue
-            $found = $true
-          }
+        foreach ($file in $logFiles) {
+          Write-Host "=== Last 200 lines of $($file.FullName) ==="
+          Get-Content $file.FullName -Tail 200
+          Copy-Item $file.FullName server-logs/ -ErrorAction SilentlyContinue
+          $found = $true
+        }
+        if (Test-Path "$env:TEMP\lemond.log") {
+          Copy-Item "$env:TEMP\lemond.log" server-logs/ -ErrorAction SilentlyContinue
+          $found = $true
         }
 
         if (-not $found) {
@@ -53,20 +50,22 @@ runs:
           done
         fi
 
-        # --- Linux: check file-based logs ---
+        # --- Linux: check file-based logs (including rotated backups) ---
         if [ "$RUNNER_OS" = "Linux" ]; then
           # XDG runtime dir (e.g. /run/user/1001/lemonade/)
           LEMON_RUNTIME_DIR="${XDG_RUNTIME_DIR:+${XDG_RUNTIME_DIR}/lemonade}"
-          for candidate in \
-            "${LEMON_RUNTIME_DIR}/lemonade-server.log" \
-            "$RUNNER_TEMP/lemonade-server.log" \
-            "$RUNNER_TEMP/lemond.log"; do
-            if [ -f "$candidate" ]; then
-              echo "=== Last 200 lines of $candidate ==="
-              tail -200 "$candidate"
-              cp "$candidate" server-logs/ 2>/dev/null || true
-              found=true
-            fi
+          for log_pattern in \
+            "${LEMON_RUNTIME_DIR}"/lemonade-server.log* \
+            "$RUNNER_TEMP"/lemonade-server.log* \
+            "$RUNNER_TEMP"/lemond.log*; do
+            for candidate in $log_pattern; do
+              if [ -f "$candidate" ]; then
+                echo "=== Last 200 lines of $candidate ==="
+                tail -200 "$candidate"
+                cp "$candidate" server-logs/ 2>/dev/null || true
+                found=true
+              fi
+            done
           done
 
           # Systemd journal (works when server runs as lemond.service)

.github/workflows/cpp_server_build_test_release.yml

@@ -1952,6 +1952,13 @@ jobs:
           .venv/bin/python test/server_websocket_auth.py
           echo "WebSocket auth tests PASSED!"
 
+      - name: Test log rotation
+        if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
+        shell: bash
+        run: |
+          .venv/bin/python -m test.utils.reset_server_state --best-effort --label "ubuntu log-rotation"
+          .venv/bin/python test/server_log_rotation.py
+
       - name: Test router
         if: ${{ !cancelled() && steps.setup.outcome == 'success' }}
         shell: bash

.github/workflows/linux_distro_builds.yml

@@ -243,6 +243,21 @@ jobs:
 
           echo "Endpoint tests PASSED!"
 
+      - name: Run log rotation tests
+        env:
+          LEMONADE_CI_MODE: "True"
+          PYTHONIOENCODING: utf-8
+        run: |
+          set -e
+
+          . .venv/bin/activate
+          CLI_BINARY="$(pwd)/build/lemonade"
+
+          echo "Running log rotation tests..."
+          python test/server_log_rotation.py
+
+          echo "Log rotation tests PASSED!"
+
       - name: Capture and upload server logs
         if: always()
         uses: ./.github/actions/capture-server-logs

CMakeLists.txt

@@ -3016,6 +3016,13 @@ if(BUILD_TESTING AND EXISTS "${_CLI_RUNTIME_OVERRIDE_TEST_SRC}")
     add_cpp_ci_test(CliRuntimeOverrideTest CI ON COMMAND test_cli_runtime_override)
 endif()
 
+set(_LOG_ROTATION_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_log_rotation.cpp")
+if(BUILD_TESTING AND EXISTS "${_LOG_ROTATION_TEST_SRC}")
+    add_executable(test_log_rotation test/cpp/test_log_rotation.cpp)
+    target_link_libraries(test_log_rotation PRIVATE lemonade-server-core)
+    add_cpp_ci_test(LogRotationTest CI ON COMMAND test_log_rotation)
+endif()
+
 # ROCm root resolution (ROCM_PATH / rocm-sdk / /opt/rocm priority): covers the
 # external-ROCm detection that lets Lemonade skip the bundled TheRock download.
 set(_ROCM_ROOT_TEST_SRC "${CMAKE_CURRENT_SOURCE_DIR}/test/cpp/test_rocm_root_resolution.cpp")

docs/dev/getting-started.md

@@ -826,7 +826,7 @@ The tray application provides a system tray icon for desktop users:
 ### Logging and Console Output
 
 When running `LemonadeServer.exe` or `lemond`:
-- **Log File:** Direct runs write logs to a persistent log file (default: `%TEMP%\lemonade-server.log` on Windows). When `lemond` runs as the systemd service, logs go to the journal instead.
+- **Log File:** Direct CLI server runs (`lemond`) default to standard console output (`stdout`/`stderr`) with file logging disabled (`"auto"` mode), allowing systemd, launchd, container runtimes, and CI runners to capture and rotate logs natively. When running via `LemonadeServer.exe` (embedded tray app) or when file logging is explicitly enabled (`--log-file enabled` or a custom file path), logs are written to `lemonade-server.log` (under `%TEMP%` on Windows, `$XDG_RUNTIME_DIR/lemonade/` on Linux) with automatic log rotation (10 MB file cap, 5 rotated backups `.1` through `.5`, bounding steady-state log disk usage to ~60 MB under normal record sizes). Pre-existing legacy logs at startup are rotated into `.1` to preserve diagnostic history and pruned across subsequent rotation cycles.
 - **Logs UI:** Click "Show Logs" in the tray or use `lemonade logs` to open the desktop app's logs view
   - Connects to the server's WebSocket log stream
   - Shows retained recent log history plus live entries

docs/guide/configuration/README.md

@@ -76,7 +76,10 @@ When `lemond` starts, effective configuration is resolved by deep-merging settin
     "vulkan_args": "",
     "vulkan_bin": "builtin"
   },
+  "log_file": "auto",
   "log_level": "info",
+  "log_max_file_size_mb": 10,
+  "log_max_files": 5,
   "max_loaded_models": 1,
   "models_dir": "auto",
   "moonshine": {
@@ -188,6 +191,9 @@ When `lemond` starts, effective configuration is resolved by deep-merging settin
 | `port` | int | 13305 | Port number for the HTTP server |
 | `host` | string | "localhost" | Address to bind for connections |
 | `log_level` | string | "info" | Logging level (trace, debug, info, warning, error, fatal, none) |
+| `log_file` | string | "auto" | File logging mode: "auto" (console-only for direct server runs, lemonade-server.log for embedded tray app), "disabled", "enabled", or custom target file path |
+| `log_max_file_size_mb` | int | 10 | Max active log file size in MB before triggering rotation (steady-state footprint bounded to ~`log_max_file_size_mb * (log_max_files + 1)`) |
+| `log_max_files` | int | 5 | Max number of rotated log backup files to retain (.1 through .N); legacy oversized files are rotated into .1 and pruned over cycles |
 | `global_timeout` | int | 600 | Timeout in seconds for HTTP, inference, and readiness checks |
 | `max_loaded_models` | int | 1 | Max models per type slot. Use -1 for unlimited |
 | `broadcast` | bool | true | Enable or disable UDP broadcasting for server discovery |

src/cpp/include/lemon/cli_parser.h

@@ -12,6 +12,9 @@ struct ServerConfig {
     int port = -1;             // -1 = not specified on CLI, use config.json value
     std::string host;          // Empty = not specified on CLI, use config.json value
     std::optional<bool> broadcast;        // std::nullopt = not specified on CLI, use config.json value
+    std::string log_file;                 // Empty = not specified on CLI, use config.json value
+    int log_max_file_size_mb = -1;        // -1 = not specified on CLI, use config.json value
+    int log_max_files = -1;               // -1 = not specified on CLI, use config.json value
 };
 
 class CLIParser {

src/cpp/include/lemon/logging_config.h

@@ -1,5 +1,9 @@
 #pragma once
 
+#include "lemon/utils/aixlog.hpp"
+
+#include <fstream>
+#include <mutex>
 #include <optional>
 #include <string>
 
@@ -10,15 +14,47 @@ enum class LoggingMode {
     embedded_tray_server,
 };
 
+struct LogRotationConfig {
+    std::string file_mode = "auto";
+    size_t max_file_size_mb = 10;
+    size_t max_files = 5;
+};
+
 struct LoggingTargets {
     bool console = false;
     bool stream_hub = true;
     bool file = false;
     std::optional<std::string> file_path;
+    LogRotationConfig rotation;
+};
+
+class RotatingFileSink : public AixLog::SinkFormat {
+public:
+    RotatingFileSink(const AixLog::Filter& filter,
+                     const std::string& filename,
+                     const std::string& format,
+                     size_t max_file_size_mb,
+                     size_t max_files);
+    ~RotatingFileSink() override;
+
+    void log(const AixLog::Metadata& metadata, const std::string& message) override;
+
+    size_t current_size() const;
+
+private:
+    void rotate_if_needed_nolock();
+    void prune_excess_backups_nolock();
+
+    std::string filename_;
+    size_t max_file_size_bytes_;
+    size_t max_files_;
+    size_t current_size_{0};
+    std::ofstream file_;
+    mutable std::mutex mutex_;
 };
 
-LoggingTargets resolve_logging_targets(LoggingMode mode);
-void configure_application_logging(const std::string& log_level, LoggingMode mode);
-void reconfigure_application_logging(const std::string& log_level);
+LoggingTargets resolve_logging_targets(LoggingMode mode, const LogRotationConfig& rotation = {});
+void configure_application_logging(const std::string& log_level, LoggingMode mode, const LogRotationConfig& rotation = {});
+void reconfigure_application_logging(const std::string& log_level, const LogRotationConfig& rotation = {});
 
 } // namespace lemon

src/cpp/include/lemon/runtime_config.h

@@ -31,6 +31,12 @@ class RuntimeConfig {
     void set_host_override(std::optional<std::string> override_val);
     int websocket_port() const;
     std::string log_level() const;
+    std::string log_file() const;
+    void set_log_file_override(std::optional<std::string> override_val);
+    int log_max_file_size_mb() const;
+    void set_log_max_file_size_mb_override(std::optional<int> override_val);
+    int log_max_files() const;
+    void set_log_max_files_override(std::optional<int> override_val);
     std::string extra_models_dir() const;
     bool broadcast() const;
     void set_broadcast_override(std::optional<bool> override_val);
@@ -158,6 +164,9 @@ class RuntimeConfig {
     std::optional<int> port_override_;
     std::optional<std::string> host_override_;
     std::optional<bool> broadcast_override_;
+    std::optional<std::string> log_file_override_;
+    std::optional<int> log_max_file_size_mb_override_;
+    std::optional<int> log_max_files_override_;
 
     // Valid log levels
     static const std::vector<std::string> valid_log_levels_;

src/cpp/include/lemon/utils/aixlog.hpp

@@ -51,6 +51,12 @@
 #endif
 
 #ifdef _WIN32
+#ifndef WIN32_LEAN_AND_MEAN
+#define WIN32_LEAN_AND_MEAN
+#endif
+#ifndef NOMINMAX
+#define NOMINMAX
+#endif
 #include <Windows.h>
 // ERROR macro is defined in Windows header
 // To avoid conflict between these macro and declaration of ERROR / DEBUG in SEVERITY enum