fail in developer mode if sync filelock is being used in async codepath

sky/server/requests/requests.py

@@ -35,6 +35,7 @@
 from sky.server.requests.serializers import encoders
 from sky.utils import asyncio_utils
 from sky.utils import common_utils
+from sky.utils import env_options
 from sky.utils import ux_utils
 from sky.utils.db import db_utils
 
@@ -51,6 +52,11 @@
 
 DEFAULT_REQUESTS_RETENTION_HOURS = 24  # 1 day
 
+
+def _is_running_pytest() -> bool:
+    return 'PYTEST_CURRENT_TEST' in os.environ
+
+
 # TODO(zhwu): For scalability, there are several TODOs:
 # [x] Have a way to queue requests.
 # [ ] Move logs to persistent place.
@@ -664,6 +670,11 @@ async def kill_request_async(request_id: str) -> bool:
 @metrics_lib.time_me
 def update_request(request_id: str) -> Generator[Optional[Request], None, None]:
     """Get and update a SkyPilot API request."""
+    # In developer mode, we assert if synchronous filelock is
+    # being used in an async context.
+    if (env_options.Options.IS_DEVELOPER.get() and not _is_running_pytest() and
+            asyncio_utils.is_running_async()):
+        assert False, 'synchronous filelock is being used in an async context'
     # Acquire the lock to avoid race conditions between multiple request
     # operations, e.g. execute and cancel.
     with filelock.FileLock(request_lock_path(request_id)):
@@ -752,6 +763,11 @@ async def get_latest_request_id_async() -> Optional[str]:
 def get_request(request_id: str,
                 fields: Optional[List[str]] = None) -> Optional[Request]:
     """Get a SkyPilot API request."""
+    # In developer mode, we assert if synchronous filelock is
+    # being used in an async context.
+    if (env_options.Options.IS_DEVELOPER.get() and not _is_running_pytest() and
+            asyncio_utils.is_running_async()):
+        assert False, 'synchronous filelock is being used in an async context'
     with filelock.FileLock(request_lock_path(request_id)):
         return _get_request_no_lock(request_id, fields)
 

sky/utils/asyncio_utils.py

@@ -76,3 +76,12 @@ async def async_wrapper(*args, **kwargs):
             raise
 
     return async_wrapper
+
+
+def is_running_async() -> bool:
+    """Check if the code is currently running inside an asyncio event loop."""
+    try:
+        asyncio.get_running_loop()
+        return True
+    except RuntimeError:
+        return False