Rebase with hypercorn 0.16.0

.gitignore

@@ -13,3 +13,5 @@ docs/reference/source/
 dist/
 .coverage
 poetry.lock
+.idea/
+.DS_Store

CHANGELOG.rst

@@ -1,3 +1,24 @@
+0.16.0 2023-01-01
+-----------------
+
+* Add a max keep alive requests configuration option, this mitigates
+  the HTTP/2 rapid reset attack.
+* Return subprocess exit code if non-zero.
+* Add ProxyFix middleware to make it easier to run Hypercorn behind a
+  proxy.
+* Support restarting workers after max requests to make it easier to
+  manage memory leaks in apps.
+* Bugfix ensure the idle task is stopped on error.
+* Bugfix revert autoreload error because reausing old sockets.
+* Bugfix send the hinted error from h11 on RemoteProtocolErrors.
+* Bugfix handle asyncio.CancelledError when socket is closed without
+  flushing.
+* Bugfix improve WSGI compliance by closing iterators, only sending
+  headers on first response byte, erroring if ``start_response`` is
+  not called, and switching wsgi.errors to stdout.
+* Don't error on LocalProtoclErrors for ws streams to better cope with
+  race conditions.
+
 0.15.0 2023-10-29
 -----------------
 

docs/discussion/dos_mitigations.rst

@@ -169,3 +169,14 @@ data that it cannot send to the client.
 
 To mitigate this Hypercorn responds to the backpressure and pauses
 (blocks) the coroutine writing the response.
+
+Rapid reset
+^^^^^^^^^^^
+
+This attack works by opening and closing streams in quick succession
+in the expectation that this is more costly for the server than the
+client.
+
+To mitigate Hypercorn will only allow a maximum number of requests per
+kept-alive connection before closing it. This ensures that cost of the
+attack is equally born by the client.

docs/how_to_guides/configuring.rst

@@ -124,6 +124,8 @@ insecure_bind              ``--insecure-bind``           The TCP host/address to
                                                          See *bind* for formatting options.
                                                          Care must be taken! See HTTP -> HTTPS
                                                          redirection docs.
+keep_alive_max_requests    N/A                           Maximum number of requests before connection    1000
+                                                         is closed. HTTP/1 & HTTP/2 only.
 keep_alive_timeout         ``--keep-alive``              Seconds to keep inactive connections alive      5s
                                                          before closing.
 keyfile                    ``--keyfile``                 Path to the SSL key file.

docs/how_to_guides/index.rst

@@ -11,6 +11,7 @@ How to guides
    dispatch_apps.rst
    http_https_redirect.rst
    logging.rst
+   proxy_fix.rst
    server_names.rst
    statsd.rst
    wsgi_apps.rst

docs/how_to_guides/proxy_fix.rst

@@ -0,0 +1,33 @@
+Fixing proxy headers
+====================
+
+If you are serving Hypercorn behind a proxy e.g. a load balancer the
+client-address, scheme, and host-header will match that of the
+connection between the proxy and Hypercorn rather than the user-agent
+(client). However, most proxies provide headers with the original
+user-agent (client) values which can be used to "fix" the headers to
+these values.
+
+Modern proxies should provide this information via a ``Forwarded``
+header from `RFC 7239
+<https://datatracker.ietf.org/doc/html/rfc7239>`_. However, this is
+rare in practice with legacy proxies using a combination of
+``X-Forwarded-For``, ``X-Forwarded-Proto`` and
+``X-Forwarded-Host``. It is important that you chose the correct mode
+(legacy, or modern) based on the proxy you use.
+
+To use the proxy fix middleware behind a single legacy proxy simply
+wrap your app and serve the wrapped app,
+
+.. code-block:: python
+
+    from hypercorn.middleware import ProxyFixMiddleware
+
+    fixed_app = ProxyFixMiddleware(app, mode="legacy", trusted_hops=1)
+
+.. warning::
+
+    The mode and number of trusted hops must match your setup or the
+    user-agent (client) may be trusted and hence able to set
+    alternative for, proto, and host values. This can, depending on
+    your usage in the app, lead to security vulnerabilities.

docs/how_to_guides/wsgi_apps.rst

@@ -9,12 +9,6 @@ Hypercorn directly serves WSGI applications:
 
     $ hypercorn module:wsgi_app
 
-.. warning::
-
-    The full response from the WSGI app will be stored in memory
-    before being sent. This prevents the WSGI app from streaming a
-    response.
-
 WSGI Middleware
 ---------------
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "Hypercorn"
-version = "0.15.0"
+version = "0.16.0"
 description = "A ASGI Server based on Hyper libraries and inspired by Gunicorn"
 authors = ["pgjones <[email protected]>"]
 classifiers = [

src/hypercorn/__main__.py

@@ -23,7 +23,7 @@ def _load_config(config_path: Optional[str]) -> Config:
         return Config.from_toml(config_path)
 
 
-def main(sys_args: Optional[List[str]] = None) -> None:
+def main(sys_args: Optional[List[str]] = None) -> int:
     parser = argparse.ArgumentParser()
     parser.add_argument(
         "application", help="The application to dispatch to as path.to.module:instance.path"
@@ -89,6 +89,19 @@ def main(sys_args: Optional[List[str]] = None) -> None:
         default=sentinel,
         type=int,
     )
+    parser.add_argument(
+        "--max-requests",
+        help="""Maximum number of requests a worker will process before restarting""",
+        default=sentinel,
+        type=int,
+    )
+    parser.add_argument(
+        "--max-requests-jitter",
+        help="This jitter causes the max-requests per worker to be "
+        "randomized by randint(0, max_requests_jitter)",
+        default=sentinel,
+        type=int,
+    )
     parser.add_argument(
         "-g", "--group", help="Group to own any unix sockets.", default=sentinel, type=int
     )
@@ -252,6 +265,10 @@ def _convert_verify_mode(value: str) -> ssl.VerifyMode:
         config.keyfile_password = args.keyfile_password
     if args.log_config is not sentinel:
         config.logconfig = args.log_config
+    if args.max_requests is not sentinel:
+        config.max_requests = args.max_requests
+    if args.max_requests_jitter is not sentinel:
+        config.max_requests_jitter = args.max_requests
     if args.pid is not sentinel:
         config.pid_path = args.pid
     if args.root_path is not sentinel:
@@ -284,8 +301,8 @@ def _convert_verify_mode(value: str) -> ssl.VerifyMode:
     if len(args.server_names) > 0:
         config.server_names = args.server_names
 
-    run(config)
+    return run(config)
 
 
 if __name__ == "__main__":
-    main()
+    sys.exit(main())

src/hypercorn/app_wrappers.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import sys
 from functools import partial
 from io import BytesIO
 from typing import Callable, List, Optional, Tuple
@@ -84,25 +85,40 @@ async def handle_http(
 
     def run_app(self, environ: dict, send: Callable) -> None:
         headers: List[Tuple[bytes, bytes]]
+        headers_sent = False
+        response_started = False
         status_code: Optional[int] = None
 
         def start_response(
             status: str,
             response_headers: List[Tuple[str, str]],
             exc_info: Optional[Exception] = None,
         ) -> None:
-            nonlocal headers, status_code
+            nonlocal headers, response_started, status_code
 
             raw, _ = status.split(" ", 1)
             status_code = int(raw)
             headers = [
                 (name.lower().encode("ascii"), value.encode("ascii"))
                 for name, value in response_headers
             ]
-            send({"type": "http.response.start", "status": status_code, "headers": headers})
+            response_started = True
 
-        for output in self.app(environ, start_response):
-            send({"type": "http.response.body", "body": output, "more_body": True})
+        response_body = self.app(environ, start_response)
+
+        if not response_started:
+            raise RuntimeError("WSGI app did not call start_response")
+
+        try:
+            for output in response_body:
+                if not headers_sent:
+                    send({"type": "http.response.start", "status": status_code, "headers": headers})
+                    headers_sent = True
+
+                send({"type": "http.response.body", "body": output, "more_body": True})
+        finally:
+            if hasattr(response_body, "close"):
+                response_body.close()
 
 
 def _build_environ(scope: HTTPScope, body: bytes) -> dict:
@@ -126,7 +142,7 @@ def _build_environ(scope: HTTPScope, body: bytes) -> dict:
         "wsgi.version": (1, 0),
         "wsgi.url_scheme": scope.get("scheme", "http"),
         "wsgi.input": BytesIO(body),
-        "wsgi.errors": BytesIO(),
+        "wsgi.errors": sys.stdout,
         "wsgi.multithread": True,
         "wsgi.multiprocess": True,
         "wsgi.run_once": False,

src/hypercorn/asyncio/run.py

@@ -4,9 +4,11 @@
 import platform
 import signal
 import ssl
+import sys
 from functools import partial
 from multiprocessing.synchronize import Event as EventType
 from os import getpid
+from random import randint
 from socket import socket
 from typing import Any, Awaitable, Callable, Optional, Set
 
@@ -30,6 +32,14 @@
 except ImportError:
     from taskgroup import Runner  # type: ignore
 
+try:
+    from asyncio import TaskGroup
+except ImportError:
+    from taskgroup import TaskGroup  # type: ignore
+
+if sys.version_info < (3, 11):
+    from exceptiongroup import BaseExceptionGroup
+
 
 def _share_socket(sock: socket) -> socket:
     # Windows requires the socket be explicitly shared across
@@ -65,7 +75,7 @@ def _signal_handler(*_: Any) -> None:  # noqa: N803
                     # Add signal handler may not be implemented on Windows
                     signal.signal(getattr(signal, signal_name), _signal_handler)
 
-        shutdown_trigger = signal_event.wait  # type: ignore
+        shutdown_trigger = signal_event.wait
 
     lifespan = Lifespan(app, config, loop)
 
@@ -84,7 +94,10 @@ def _signal_handler(*_: Any) -> None:  # noqa: N803
         ssl_context = config.create_ssl_context()
         ssl_handshake_timeout = config.ssl_handshake_timeout
 
-    context = WorkerContext()
+    max_requests = None
+    if config.max_requests is not None:
+        max_requests = config.max_requests + randint(0, config.max_requests_jitter)
+    context = WorkerContext(max_requests)
     server_tasks: Set[asyncio.Task] = set()
 
     async def _server_callback(reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None:
@@ -136,7 +149,13 @@ async def _server_callback(reader: asyncio.StreamReader, writer: asyncio.StreamW
         await config.log.info(f"Running on https://{bind} (QUIC) (CTRL + C to quit)")
 
     try:
-        await raise_shutdown(shutdown_trigger)
+        async with TaskGroup() as task_group:
+            task_group.create_task(raise_shutdown(shutdown_trigger))
+            task_group.create_task(raise_shutdown(context.terminate.wait))
+    except BaseExceptionGroup as error:
+        _, other_errors = error.split((ShutdownError, KeyboardInterrupt))
+        if other_errors is not None:
+            raise other_errors
     except (ShutdownError, KeyboardInterrupt):
         pass
     finally:

src/hypercorn/asyncio/tcp_server.py

@@ -47,10 +47,10 @@ async def run(self) -> None:
             server = parse_socket_addr(socket.family, socket.getsockname())
             ssl_object = self.writer.get_extra_info("ssl_object")
             if ssl_object is not None:
-                ssl = True
+                tls = {}
                 alpn_protocol = ssl_object.selected_alpn_protocol()
             else:
-                ssl = False
+                tls = None
                 alpn_protocol = "http/1.1"
 
             async with TaskGroup(self.loop) as task_group:
@@ -59,11 +59,12 @@ async def run(self) -> None:
                     self.config,
                     self.context,
                     task_group,
-                    ssl,
+                    tls,
                     client,
                     server,
                     self.protocol_send,
                     alpn_protocol,
+                    (self.reader, self.writer),
                 )
                 await self.protocol.initiate()
                 await self._start_idle()
@@ -115,10 +116,16 @@ async def _close(self) -> None:
         try:
             self.writer.close()
             await self.writer.wait_closed()
-        except (BrokenPipeError, ConnectionAbortedError, ConnectionResetError, RuntimeError):
+        except (
+            BrokenPipeError,
+            ConnectionAbortedError,
+            ConnectionResetError,
+            RuntimeError,
+            asyncio.CancelledError,
+        ):
             pass  # Already closed
-
-        await self._stop_idle()
+        finally:
+            await self._stop_idle()
 
     async def _initiate_server_close(self) -> None:
         await self.protocol.handle(Closed())

src/hypercorn/asyncio/worker_context.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 import asyncio
-from typing import Type, Union
+from typing import Optional, Type, Union
 
 from ..typing import Event
 
@@ -26,9 +26,20 @@ def is_set(self) -> bool:
 class WorkerContext:
     event_class: Type[Event] = EventWrapper
 
-    def __init__(self) -> None:
+    def __init__(self, max_requests: Optional[int]) -> None:
+        self.max_requests = max_requests
+        self.requests = 0
+        self.terminate = self.event_class()
         self.terminated = self.event_class()
 
+    async def mark_request(self) -> None:
+        if self.max_requests is None:
+            return
+
+        self.requests += 1
+        if self.requests > self.max_requests:
+            await self.terminate.set()
+
     @staticmethod
     async def sleep(wait: Union[float, int]) -> None:
         return await asyncio.sleep(wait)

src/hypercorn/config.py

@@ -84,13 +84,16 @@ class Config:
     include_date_header = True
     include_server_header = True
     keep_alive_timeout = 5 * SECONDS
+    keep_alive_max_requests = 1000
     keyfile: Optional[str] = None
     keyfile_password: Optional[str] = None
     logconfig: Optional[str] = None
     logconfig_dict: Optional[dict] = None
     logger_class = Logger
     loglevel: str = "INFO"
     max_app_queue_size: int = 10
+    max_requests: Optional[int] = None
+    max_requests_jitter: int = 0
     pid_path: Optional[str] = None
     server_names: List[str] = []
     shutdown_timeout = 60 * SECONDS