Cap BLAS threads to prevent fork-vs-OpenBLAS atfork freeze (#111)

glibc fork() runs OpenBLAS's pthread_atfork prepare handler, which joins
the entire BLAS worker pool. uvloop spawns agent CLIs via fork() on the
event loop while the memU thread keeps the pool busy with vector-index
mat-vecs — under recall load the pool never quiesces and the loop freezes
inside fork() for minutes. Default OPENBLAS_NUM_THREADS=1 (no pool, no
atfork wait; our mat-vecs are bandwidth-bound anyway) and skip the SDK's
per-connect `claude -v` subprocess. Explicit env always wins.

Author: pufit

nerve/_env.py

@@ -0,0 +1,61 @@
+"""Process-wide environment defaults — import before numpy loads.
+
+This module is imported for its side effects at the top of Nerve's entry
+points (``nerve.cli``) and before the first ``import numpy`` in the
+codebase (``nerve.memory.memu_bridge``).  BLAS thread-pool sizing is read
+once when the BLAS shared library is loaded, so these defaults are only
+effective if they are in the environment *before* numpy is imported
+anywhere in the process.
+
+Why cap BLAS at a single thread
+===============================
+
+OpenBLAS (bundled with numpy wheels) spawns a worker thread pool sized to
+the machine's core count on first parallel operation.  On many-core hosts
+this interacts catastrophically with subprocess spawning:
+
+1. **fork vs. BLAS atfork collision.**  glibc ``fork()`` runs registered
+   ``pthread_atfork`` prepare handlers.  OpenBLAS registers one
+   (``blas_thread_shutdown_``) that ``pthread_join``\\ s its *entire*
+   worker pool before allowing the fork to proceed.  The event loop
+   spawns agent CLI subprocesses via ``fork()`` (libuv/uvloop), while the
+   dedicated memU thread keeps the BLAS pool busy with vector-index
+   mat-vecs over a multi-hundred-MB embedding matrix.  Under a recall
+   storm the pool never quiesces, so a spawn on the loop thread can block
+   for minutes inside ``fork()`` — freezing the entire server (HTTP,
+   WebSocket, all sessions).  Observed in production on a many-core
+   deployment; diagnosed via native stack: ``uv__spawn_and_init_child_fork
+   → __libc_fork → __run_prefork_handlers → blas_thread_shutdown_ →
+   pthread_join``.
+
+2. **Multi-threaded BLAS buys us nothing.**  Nerve's only BLAS workload
+   is one mat-vec per memory recall/dedup — memory-bandwidth-bound, ~tens
+   of ms single-threaded even on large indexes.
+
+With ``OPENBLAS_NUM_THREADS=1`` the pool is never created, the atfork
+handler returns immediately, and forks never stall.
+
+``os.environ.setdefault`` is used throughout: an explicitly configured
+environment always wins over these defaults.
+"""
+
+from __future__ import annotations
+
+import os
+
+
+def apply_env_defaults() -> None:
+    """Apply process-env defaults.  Idempotent; explicit env wins."""
+    # Cap the BLAS worker pool (see module docstring).  Must be set
+    # before numpy/OpenBLAS loads.
+    os.environ.setdefault("OPENBLAS_NUM_THREADS", "1")
+
+    # The Claude Agent SDK spawns an extra ``claude -v`` subprocess on
+    # every connect just to warn about outdated CLIs.  With many
+    # concurrent sessions this doubles fork pressure on the event loop
+    # for a warning nothing acts on.  The SDK honors this variable to
+    # skip the check; export it as an empty string to re-enable.
+    os.environ.setdefault("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "1")
+
+
+apply_env_defaults()

nerve/cli.py

@@ -14,6 +14,10 @@
 
 from __future__ import annotations
 
+# Must be first: applies BLAS thread caps and other process-env defaults
+# that have to be in place before numpy (or any BLAS user) is imported.
+import nerve._env  # noqa: F401  isort: skip
+
 import asyncio
 import logging
 import os

nerve/memory/memu_bridge.py

@@ -20,6 +20,12 @@
 from typing import Any, Coroutine
 from zoneinfo import ZoneInfo
 
+# Ensure BLAS thread caps are applied before numpy loads OpenBLAS — an
+# unbounded BLAS pool makes glibc fork() (subprocess spawning on the event
+# loop) block in OpenBLAS's atfork handler while the memU thread runs
+# vector searches.  See nerve/_env.py for the full mechanism.
+import nerve._env  # noqa: F401  isort: skip
+
 import numpy as np
 
 from nerve.config import NerveConfig

tests/test_env_defaults.py

@@ -0,0 +1,58 @@
+"""Tests for nerve._env — process-env defaults applied before numpy loads.
+
+These defaults prevent the fork-vs-OpenBLAS-atfork collision: an unbounded
+BLAS worker pool makes glibc fork() (used by uvloop to spawn agent CLIs on
+the event loop) block in OpenBLAS's pthread_atfork prepare handler while
+the memU thread runs vector searches.
+"""
+
+from __future__ import annotations
+
+import importlib
+import os
+
+import nerve._env as env_mod
+
+
+class TestEnvDefaults:
+    def test_defaults_applied_when_unset(self, monkeypatch):
+        monkeypatch.delenv("OPENBLAS_NUM_THREADS", raising=False)
+        monkeypatch.delenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", raising=False)
+
+        env_mod.apply_env_defaults()
+
+        assert os.environ["OPENBLAS_NUM_THREADS"] == "1"
+        assert os.environ["CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK"] == "1"
+
+    def test_explicit_values_win(self, monkeypatch):
+        monkeypatch.setenv("OPENBLAS_NUM_THREADS", "8")
+        # Empty string re-enables the SDK version check (falsy in the SDK).
+        monkeypatch.setenv("CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK", "")
+
+        env_mod.apply_env_defaults()
+
+        assert os.environ["OPENBLAS_NUM_THREADS"] == "8"
+        assert os.environ["CLAUDE_AGENT_SDK_SKIP_VERSION_CHECK"] == ""
+
+    def test_applied_on_import(self, monkeypatch):
+        """Importing the module applies the defaults (entry-point contract)."""
+        monkeypatch.delenv("OPENBLAS_NUM_THREADS", raising=False)
+        importlib.reload(env_mod)
+        assert os.environ["OPENBLAS_NUM_THREADS"] == "1"
+
+    def test_idempotent(self, monkeypatch):
+        monkeypatch.delenv("OPENBLAS_NUM_THREADS", raising=False)
+        env_mod.apply_env_defaults()
+        env_mod.apply_env_defaults()
+        assert os.environ["OPENBLAS_NUM_THREADS"] == "1"
+
+    def test_bridge_import_applies_caps(self):
+        """memu_bridge must guarantee the caps before its numpy import.
+
+        numpy is typically already loaded by the time this test runs, so
+        this can't verify load-order end-to-end — it pins the import
+        dependency: importing the bridge module must (re)apply defaults.
+        """
+        import nerve.memory.memu_bridge  # noqa: F401
+
+        assert os.environ.get("OPENBLAS_NUM_THREADS") is not None