Allow unlimited connections per-worker

Signed-off-by: Samuel Monson <[email protected]>

Author: sjmonson

src/guidellm/backends/openai.py

@@ -134,6 +134,12 @@ async def process_startup(self):
             timeout=self.timeout,
             follow_redirects=self.follow_redirects,
             verify=self.verify,
+            # Allow unlimited connections with short keep-alive
+            limits=httpx.Limits(
+                max_connections=None,
+                max_keepalive_connections=None,
+                keepalive_expiry=0.5,
+            ),
         )
         self._in_process = True