From eb38e357d9a31ea23708a699ae87ea679e435181 Mon Sep 17 00:00:00 2001 From: Jody Clements Date: Fri, 23 Jan 2026 11:05:18 -0500 Subject: [PATCH] fix: Prevents concurrent alembic migrations with file-based locking When running uvicorn with multiple workers (--workers 10), all worker processes were attempting to run alembic migrations simultaneously, causing race conditions and errors. Implemented file-based locking using fcntl to ensure only one process runs migrations while other workers wait. The lock file is created in the system temp directory with a hash of the database URL to support multiple databases. Also removed the redundant global _migrations_run flag which only prevented duplicate runs within a single process, not across multiple worker processes. Added test-multi-worker task to pyproject.toml for testing with 10 workers. --- fileglancer/database.py | 120 +++++++++++++++++++++++----------------- pixi.lock | 10 +--- pyproject.toml | 1 + 3 files changed, 72 insertions(+), 59 deletions(-) diff --git a/fileglancer/database.py b/fileglancer/database.py index 21616aac..939616fd 100644 --- a/fileglancer/database.py +++ b/fileglancer/database.py @@ -2,6 +2,8 @@ import hashlib from datetime import datetime, UTC import os +import fcntl +import tempfile from functools import lru_cache from sqlalchemy import create_engine, Column, String, Integer, DateTime, JSON, UniqueConstraint @@ -20,9 +22,6 @@ SHARING_KEY_LENGTH = 12 NEUROGLANCER_SHORT_KEY_LENGTH = 12 -# Global flag to track if migrations have been run -_migrations_run = False - # Engine cache - maintain multiple engines for different database URLs _engine_cache = {} @@ -154,58 +153,77 @@ class SessionDB(Base): def run_alembic_upgrade(db_url): """Run Alembic migrations to upgrade database to latest version""" - global _migrations_run + # Use a file lock to ensure only one process runs migrations + # Hash the db_url to create a unique lock file per database + db_hash = hashlib.sha256(db_url.encode()).hexdigest()[:16] + lock_file_path = os.path.join(tempfile.gettempdir(), f"fileglancer_migration_{db_hash}.lock") - if _migrations_run: - logger.debug("Migrations already run, skipping") - return + logger.debug(f"Attempting to acquire migration lock: {lock_file_path}") + lock_file = open(lock_file_path, 'w') try: - from alembic.config import Config - from alembic import command - import os - - alembic_cfg_path = None - - # Try to find alembic.ini - first in package directory, then development setup - current_dir = os.path.dirname(os.path.abspath(__file__)) - - # Check if alembic.ini is in the package directory (installed package) - pkg_alembic_cfg_path = os.path.join(current_dir, "alembic.ini") - if os.path.exists(pkg_alembic_cfg_path): - alembic_cfg_path = pkg_alembic_cfg_path - logger.debug("Using packaged alembic.ini") - else: - # Fallback to development setup - project_root = os.path.dirname(current_dir) - dev_alembic_cfg_path = os.path.join(project_root, "alembic.ini") - if os.path.exists(dev_alembic_cfg_path): - alembic_cfg_path = dev_alembic_cfg_path - logger.debug("Using development alembic.ini") - - if alembic_cfg_path and os.path.exists(alembic_cfg_path): - alembic_cfg = Config(alembic_cfg_path) - alembic_cfg.set_main_option("sqlalchemy.url", db_url) - - # Update script_location for packaged installations - if alembic_cfg_path == pkg_alembic_cfg_path: - # Using packaged alembic.ini, also update script_location - pkg_alembic_dir = os.path.join(current_dir, "alembic") - if os.path.exists(pkg_alembic_dir): - alembic_cfg.set_main_option("script_location", pkg_alembic_dir) - - command.upgrade(alembic_cfg, "head") - logger.info("Alembic migrations completed successfully") - else: - logger.warning("Alembic configuration not found, falling back to create_all") - engine = _get_engine(db_url) - Base.metadata.create_all(engine) - except Exception as e: - logger.warning(f"Alembic migration failed, falling back to create_all: {e}") - engine = _get_engine(db_url) - Base.metadata.create_all(engine) + # Try to acquire an exclusive lock (non-blocking first to check) + try: + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX | fcntl.LOCK_NB) + logger.info("Acquired migration lock, running migrations") + lock_acquired = True + except BlockingIOError: + # Another process is running migrations, wait for it + logger.info("Another process is running migrations, waiting...") + fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX) + logger.info("Migration lock acquired after waiting, migrations should already be complete") + lock_acquired = False + + # Only run migrations if we were the first to acquire the lock + if lock_acquired: + try: + from alembic.config import Config + from alembic import command + + alembic_cfg_path = None + + # Try to find alembic.ini - first in package directory, then development setup + current_dir = os.path.dirname(os.path.abspath(__file__)) + + # Check if alembic.ini is in the package directory (installed package) + pkg_alembic_cfg_path = os.path.join(current_dir, "alembic.ini") + if os.path.exists(pkg_alembic_cfg_path): + alembic_cfg_path = pkg_alembic_cfg_path + logger.debug("Using packaged alembic.ini") + else: + # Fallback to development setup + project_root = os.path.dirname(current_dir) + dev_alembic_cfg_path = os.path.join(project_root, "alembic.ini") + if os.path.exists(dev_alembic_cfg_path): + alembic_cfg_path = dev_alembic_cfg_path + logger.debug("Using development alembic.ini") + + if alembic_cfg_path and os.path.exists(alembic_cfg_path): + alembic_cfg = Config(alembic_cfg_path) + alembic_cfg.set_main_option("sqlalchemy.url", db_url) + + # Update script_location for packaged installations + if alembic_cfg_path == pkg_alembic_cfg_path: + # Using packaged alembic.ini, also update script_location + pkg_alembic_dir = os.path.join(current_dir, "alembic") + if os.path.exists(pkg_alembic_dir): + alembic_cfg.set_main_option("script_location", pkg_alembic_dir) + + command.upgrade(alembic_cfg, "head") + logger.info("Alembic migrations completed successfully") + else: + logger.warning("Alembic configuration not found, falling back to create_all") + engine = _get_engine(db_url) + Base.metadata.create_all(engine) + except Exception as e: + logger.warning(f"Alembic migration failed, falling back to create_all: {e}") + engine = _get_engine(db_url) + Base.metadata.create_all(engine) finally: - _migrations_run = True + # Release the lock + fcntl.flock(lock_file.fileno(), fcntl.LOCK_UN) + lock_file.close() + logger.debug("Released migration lock") def initialize_database(db_url): diff --git a/pixi.lock b/pixi.lock index f65b58ce..e4b24d22 100644 --- a/pixi.lock +++ b/pixi.lock @@ -5,8 +5,6 @@ environments: - url: https://conda.anaconda.org/conda-forge/ indexes: - https://pypi.org/simple - options: - pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -508,8 +506,6 @@ environments: - url: https://conda.anaconda.org/conda-forge/ indexes: - https://pypi.org/simple - options: - pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -1127,8 +1123,6 @@ environments: - url: https://conda.anaconda.org/conda-forge/ indexes: - https://pypi.org/simple - options: - pypi-prerelease-mode: if-necessary-or-explicit packages: linux-64: - conda: https://conda.anaconda.org/conda-forge/linux-64/_libgcc_mutex-0.1-conda_forge.tar.bz2 @@ -2996,8 +2990,8 @@ packages: timestamp: 1760972937564 - pypi: ./ name: fileglancer - version: 2.2.1 - sha256: 0128c34e0ae7fe36a12ca9c59571488b221877ac4b330783a3daaa722312af1d + version: 2.4.0 + sha256: bfbde34083039b00f72e03149635389893177139077a5e3547ada6c65dca7da0 requires_dist: - aiosqlite>=0.21.0 - alembic>=1.17.0 diff --git a/pyproject.toml b/pyproject.toml index 94805d90..26fc37c6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -143,6 +143,7 @@ dev-watch = { cmd = "cd frontend && NODE_ENV=development npm run watch" } dev-launch = "pixi run uvicorn fileglancer.app:app --no-access-log --port 7878 --reload" dev-launch-remote = "pixi run uvicorn fileglancer.app:app --host 0.0.0.0 --port 7878 --reload --ssl-keyfile /opt/certs/cert.key --ssl-certfile /opt/certs/cert.crt" prod-launch-remote = "pixi run uvicorn fileglancer.app:app --workers 10 --host 0.0.0.0 --port 7878 --ssl-keyfile /opt/certs/cert.key --ssl-certfile /opt/certs/cert.crt" +test-multi-worker = "pixi run uvicorn fileglancer.app:app --host 127.0.0.1 --workers 10 --port 8989 --no-access-log" dev-launch-secure = "python fileglancer/dev_launch.py" migrate = "alembic upgrade head" migrate-create = "alembic revision --autogenerate"