Merge branch 'release/4.5.21'

Author: GrahamDumpleton

README.rst

@@ -359,7 +359,7 @@ MacOS X Sierra, the System Integrity Protection (SIP) system of MacOS X,
 prevented installing the mod_wsgi module into the Apache modules directory.
 
 If you are using Windows, the CMMI method was never supported as Windows
-doesn't supply the required tools tools to make it work.
+doesn't supply the required tools to make it work.
 
 The CMMI installation method also involves a bit more work as you need to
 separately download the mod_wsgi source code, run the ``configure`` tool

docs/release-notes.rst

@@ -5,6 +5,7 @@ Release Notes
 .. toctree::
    :maxdepth: 2
 
+   release-notes/version-4.5.21
    release-notes/version-4.5.20
    release-notes/version-4.5.19
    release-notes/version-4.5.18

docs/release-notes/version-4.5.21.rst

@@ -0,0 +1,46 @@
+==============
+Version 4.5.21
+==============
+
+Version 4.5.21 of mod_wsgi can be obtained from:
+
+  https://codeload.github.com/GrahamDumpleton/mod_wsgi/tar.gz/4.5.21
+
+Bugs Fixed
+----------
+
+* Installation using ``pip`` or ``setup.py`` install was failing on
+  MacOS X High Sierra with latest Xcode as Apple doesn't even include
+  the ``apxs`` program at all.
+
+  Note you still cannot use the configure/make/make install method of
+  deploying mod_wsgi to MacOS X. You need to use the ``pip install``
+  method.
+
+* Speculated that crashes on daemon process shutdown were being caused
+  by a race condition around accessing Python C API when interpreter
+  was being destroyed. There was a check in place to avoid this but may
+  not have been robust enough depending on how memory cache worked
+  for threads running across multi core machine. Now use a dedicated
+  thread mutex to avoid race condition between main process thread and
+  Python interpreter deadlock detection thread.
+
+Features Changed
+----------------
+
+* Set ``wsgi.input_terminated`` to ``True`` in WSGI environment. This is a
+  unofficial extension to WSGI specification proposed by Armin Ronacher
+  for a WSGI server/middleware to flag that it is safe to read to the
+  end of input and that ``CONTENT_LENGTH`` can be ignored. This is to be
+  able to support chunked request content, but also anything which
+  mutates the request content length but which can't easily change the
+  ``CONTENT_LENGTH``, such as occurs when request content is compressed
+  and is decompressed by the Apache web server.
+
+  The ability to safely read until end of input was always present in
+  mod_wsgi, but there was no way in the WSGI specification for a WSGI
+  server to tell a WSGI application this was the case. Prior attempts to
+  include something to deal with this in the WSGI specification when it
+  was updated in PEP 3333 were ignored. This is why now an unofficial way
+  of doing it is being adopted by WSGI servers separate to the WSGI
+  specification.

setup.py

@@ -133,6 +133,7 @@ def find_program(names, default=None, paths=[]):
 
 WITHOUT_APXS = False
 WITH_WINDOWS_APACHE = None
+WITH_MACOSX_APACHE = None
 
 if not WITH_TARBALL_PACKAGE:
     if not os.path.isabs(APXS) or not os.access(APXS, os.X_OK):
@@ -157,7 +158,10 @@ def find_program(names, default=None, paths=[]):
             raise RuntimeError('No Apache installation can be found. Set the '
                     'MOD_WSGI_APACHE_ROOTDIR environment to its location.')
 
-if WITHOUT_APXS and not WITH_WINDOWS_APACHE:
+elif WITHOUT_APXS and sys.platform == 'darwin':
+    WITH_MACOSX_APACHE = '/Applications/Xcode.app'
+
+if WITHOUT_APXS and not WITH_WINDOWS_APACHE and not WITHOUT_APXS:
     raise RuntimeError('The %r command appears not to be installed or '
             'is not executable. Please check the list of prerequisites '
             'in the documentation for this package and install any '
@@ -178,6 +182,27 @@ def get_apr_includes():
     def get_apu_includes():
         return ''
 
+elif WITH_MACOSX_APACHE:
+    def get_apxs_config(name):
+        if name == 'BINDIR':
+            return '/usr/bin'
+        elif name == 'SBINDIR':
+            return '/usr/sbin'
+        elif name == 'LIBEXECDIR':
+            return '/usr/libexec/apache2'
+        elif name == 'PROGNAME':
+            return 'httpd'
+        elif name == 'SHLIBPATH_VAR':
+            return 'DYLD_LIBRARY_PATH'
+        else:
+            return ''
+
+    def get_apr_includes():
+        return ''
+
+    def get_apu_includes():
+        return ''
+
 elif WITH_TARBALL_PACKAGE:
     SCRIPT_DIR = os.path.join(os.path.dirname(__file__), 'src', 'packages')
 

src/server/mod_wsgi.c

@@ -78,7 +78,7 @@ static apr_array_header_t *wsgi_daemon_list = NULL;
 
 static apr_pool_t *wsgi_parent_pool = NULL;
 
-static int volatile wsgi_daemon_shutdown = 0;
+int volatile wsgi_daemon_shutdown = 0;
 static int volatile wsgi_daemon_graceful = 0;
 
 #if defined(MOD_WSGI_WITH_DAEMONS)
@@ -2708,6 +2708,8 @@ static PyObject *Adapter_environ(AdapterObject *self)
     object = (PyObject *)self->input;
     PyDict_SetItemString(vars, "wsgi.input", object);
 
+    PyDict_SetItemString(vars, "wsgi.input_terminated", Py_True);
+
     /* Setup file wrapper object for efficient file responses. */
 
     PyDict_SetItemString(vars, "wsgi.file_wrapper", (PyObject *)&Stream_Type);
@@ -4286,6 +4288,7 @@ static void wsgi_python_child_init(apr_pool_t *p)
 #if APR_HAS_THREADS
     apr_thread_mutex_create(&wsgi_interp_lock, APR_THREAD_MUTEX_UNNESTED, p);
     apr_thread_mutex_create(&wsgi_module_lock, APR_THREAD_MUTEX_UNNESTED, p);
+    apr_thread_mutex_create(&wsgi_shutdown_lock, APR_THREAD_MUTEX_UNNESTED, p);
 #endif
 
     /*
@@ -8952,11 +8955,15 @@ static void *wsgi_deadlock_thread(apr_thread_t *thd, void *data)
     while (1) {
         apr_sleep(apr_time_from_sec(1));
 
+        apr_thread_mutex_lock(wsgi_shutdown_lock);
+
         if (!wsgi_daemon_shutdown) {
             gilstate = PyGILState_Ensure();
             PyGILState_Release(gilstate);
         }
 
+        apr_thread_mutex_unlock(wsgi_shutdown_lock);
+
         apr_thread_mutex_lock(wsgi_monitor_lock);
         wsgi_deadlock_shutdown_time = apr_time_now();
         wsgi_deadlock_shutdown_time += wsgi_deadlock_timeout;

src/server/wsgi_daemon.h

@@ -185,6 +185,8 @@ extern WSGIDaemonThread *wsgi_worker_threads;
 
 extern WSGIThreadStack *wsgi_worker_stack;
 
+extern int volatile wsgi_daemon_shutdown;
+
 #endif
 
 /* ------------------------------------------------------------------------- */

src/server/wsgi_interp.c

@@ -1986,10 +1986,22 @@ apr_status_t wsgi_python_term(void)
     if (!PyImport_AddModule("dummy_threading"))
         PyErr_Clear();
 
-    /* Shutdown Python interpreter completely. */
+    /*
+     * Shutdown Python interpreter completely. Just to be safe
+     * flag daemon shutdown here again and do it within a lock
+     * which is then shared with deadlock thread used for the
+     * daemon. This is just to avoid any risk there is a race
+     * condition.
+     */
+
+    apr_thread_mutex_lock(wsgi_shutdown_lock);
+
+    wsgi_daemon_shutdown++;
 
     Py_Finalize();
 
+    apr_thread_mutex_unlock(wsgi_shutdown_lock);
+
     wsgi_python_initialized = 0;
 
     ap_log_error(APLOG_MARK, APLOG_INFO, 0, wsgi_server,
@@ -2336,6 +2348,7 @@ void wsgi_python_init(apr_pool_t *p)
 
 #if APR_HAS_THREADS
 apr_thread_mutex_t* wsgi_interp_lock = NULL;
+apr_thread_mutex_t* wsgi_shutdown_lock = NULL;
 #endif
 
 PyObject *wsgi_interpreters = NULL;

src/server/wsgi_interp.h

@@ -70,6 +70,7 @@ extern PyObject *wsgi_interpreters;
 
 #if APR_HAS_THREADS
 extern apr_thread_mutex_t *wsgi_interp_lock;
+extern apr_thread_mutex_t* wsgi_shutdown_lock;
 #endif
 
 extern void wsgi_python_version(void);

src/server/wsgi_version.h

@@ -25,8 +25,8 @@
 
 #define MOD_WSGI_MAJORVERSION_NUMBER 4
 #define MOD_WSGI_MINORVERSION_NUMBER 5
-#define MOD_WSGI_MICROVERSION_NUMBER 20
-#define MOD_WSGI_VERSION_STRING "4.5.20"
+#define MOD_WSGI_MICROVERSION_NUMBER 21
+#define MOD_WSGI_VERSION_STRING "4.5.21"
 
 /* ------------------------------------------------------------------------- */