Introduce end2end test framework (#222)

Author: wkeeling

.flake8

@@ -1,5 +1,5 @@
 [flake8]
-exclude = docs
+exclude = docs, thirdparty
 max-line-length = 119
 per-file-ignores =
     */__init__.py: F401, F403

.github/workflows/cibuild.yml

@@ -4,26 +4,51 @@ on:
   push:
     branches:
       - master
-      - gh_actions
+      - end2end_tests
 
 jobs:
-  tox:
+  flake8:
     runs-on: ubuntu-latest
-    strategy:
-      matrix:
-        python-version: [3.6, 3.7, 3.8]
     steps:
     - uses: actions/checkout@v1
+    - uses: actions/setup-python@v2
+    - run: python -m pip install --upgrade pip
+    - run: pip install tox
+    - run: tox -e flake8
 
-    - name: Setup Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install tox tox-gh-actions
+  unit_tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - os: windows-latest
+            py: 3.9
+          - os: macos-latest
+            py: 3.9
+          - os: ubuntu-latest
+            py: 3.9
+          - os: ubuntu-latest
+            py: 3.8
+          - os: ubuntu-latest
+            py: 3.7
+          - os: ubuntu-latest
+            py: 3.6
+    runs-on: ${{ matrix.os }}
+    steps:
+      - run: printenv
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.py }}
+      - run: pip install tox
+      - run: tox -e py
 
-    - name: Run tox
-      run: tox
+  end2end_tests:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v1
+      - uses: actions/setup-python@v2
+        with:
+          python-version: 3.9
+      - run: pip install tox
+      - run: tox -e e2e

.gitignore

@@ -46,6 +46,7 @@ coverage.xml
 *.cover
 .hypothesis/
 .pytest_cache/
+headless-chromium
 
 # Translations
 *.mo

README.rst

@@ -9,7 +9,7 @@ Selenium Wire extends Selenium's Python bindings to give you access to the under
 .. image:: https://codecov.io/gh/wkeeling/selenium-wire/branch/master/graph/badge.svg
         :target: https://codecov.io/gh/wkeeling/selenium-wire
 
-.. image:: https://img.shields.io/badge/python-3.6%2C%203.7%2C%203.8-blue.svg
+.. image:: https://img.shields.io/badge/python-3.6%2C%203.7%2C%203.8%2C%203.9-blue.svg
         :target: https://pypi.python.org/pypi/selenium-wire
 
 .. image:: https://img.shields.io/pypi/v/selenium-wire.svg
@@ -480,7 +480,7 @@ Selenium Wire works by redirecting browser traffic through an internal proxy ser
         driver = webdriver.Firefox(seleniumwire_options=options)
 
 ``request.abort()``
-    You can abort a request early by using ``request.abort()`` from within a `request interceptor`_. This will send an immediate response back to the client without the request travelling any further. You can use this mechanism to block certain types of requests (e.g. images) to improve page load performance. Aborted requests are not captured.
+    You can abort a request early by using ``request.abort()`` from within a `request interceptor`_. This will send an immediate response back to the client without the request travelling any further. You can use this mechanism to block certain types of requests (e.g. images) to improve page load performance.
 
     .. code:: python
 
@@ -507,7 +507,7 @@ The configuration takes the following format:
     options = {
         'proxy': {
             'http': 'http://192.168.10.100:8888',
-            'https': 'https://192.168.10.100:8889',
+            'https': 'https://192.168.10.100:8888',
             'no_proxy': 'localhost,127.0.0.1'
         }
     }
@@ -519,17 +519,17 @@ To use HTTP Basic Auth with your proxy, specify the username and password in the
 
     options = {
         'proxy': {
-            'https': 'https://user:[email protected]:8889',
+            'https': 'https://user:[email protected]:8888',
         }
     }
 
-For proxy authentication different to Basic, you can supply the full value for the ``Proxy-Authorization`` header using the ``custom_authorization`` option. For example, if your proxy used the Bearer scheme:
+For authentication other than Basic, you can supply the full value for the ``Proxy-Authorization`` header using the ``custom_authorization`` option. For example, if your proxy used the Bearer scheme:
 
 .. code:: python
 
     options = {
         'proxy': {
-            'https': 'https://192.168.10.100:8889',  # No username or password used
+            'https': 'https://192.168.10.100:8888',  # No username or password used
             'custom_authorization': 'Bearer mytoken123'  # Custom Proxy-Authorization header value
         }
     }
@@ -543,7 +543,7 @@ The proxy configuration can also be loaded through environment variables called
 .. code:: bash
 
     $ export HTTP_PROXY="http://192.168.10.100:8888"
-    $ export HTTPS_PROXY="https://192.168.10.100:8889"
+    $ export HTTPS_PROXY="https://192.168.10.100:8888"
     $ export NO_PROXY="localhost,127.0.0.1"
 
 SOCKS
@@ -556,7 +556,7 @@ Using a SOCKS proxy is the same as using an HTTP based one:
     options = {
         'proxy': {
             'http': 'socks5://user:[email protected]:8888',
-            'https': 'socks5://user:[email protected]:8889',
+            'https': 'socks5://user:[email protected]:8888',
             'no_proxy': 'localhost,127.0.0.1'
         }
     }

seleniumwire/handler.py

@@ -44,12 +44,11 @@ def request(self, flow):
                     content=request.response.body,
                     headers=[(k.encode('utf-8'), v.encode('utf-8')) for k, v in request.response.headers.items()]
                 )
-                return
-
-            flow.request.method = request.method
-            flow.request.url = request.url
-            flow.request.headers = self._to_headers_obj(request.headers)
-            flow.request.raw_content = request.body
+            else:
+                flow.request.method = request.method
+                flow.request.url = request.url
+                flow.request.headers = self._to_headers_obj(request.headers)
+                flow.request.raw_content = request.body
 
         log.info('Capturing request: %s', request.url)
 
@@ -58,6 +57,10 @@ def request(self, flow):
         if request.id is not None:  # Will not be None when captured
             flow.request.id = request.id
 
+        if request.response:
+            # This response will be a mocked response. Capture it for completeness.
+            self.proxy.storage.save_response(request.id, request.response)
+
         # Could possibly use mitmproxy's 'anticomp' option instead of this
         if self.proxy.options.get('disable_encoding') is True:
             flow.request.headers['Accept-Encoding'] = 'identity'

seleniumwire/inspect.py

@@ -1,6 +1,6 @@
 import inspect
 import time
-from typing import List, Optional
+from typing import List, Optional, Union
 
 from selenium.common.exceptions import TimeoutException
 
@@ -40,7 +40,7 @@ def last_request(self) -> Optional[Request]:
         """
         return self.proxy.storage.load_last_request()
 
-    def wait_for_request(self, path: str, timeout: int = 10) -> Request:
+    def wait_for_request(self, path: str, timeout: Union[int, float] = 10) -> Request:
         """Wait up to the timeout period for a request with the specified
         path to be seen.
 
@@ -68,7 +68,7 @@ def wait_for_request(self, path: str, timeout: int = 10) -> Request:
             request = self.proxy.storage.find(path)
 
             if request is None:
-                time.sleep(0.2)
+                time.sleep(1 / 5)
             else:
                 return request
 

seleniumwire/mitmproxy.py

@@ -292,4 +292,4 @@ class SendToLogger:
 
     def log(self, entry):
         """Send a mitmproxy log message through our own logger."""
-        getattr(logger, entry.level, logger.info)(entry.msg)
+        getattr(logger, entry.level.replace('warn', 'warning'), logger.info)(entry.msg)

setup.py

@@ -53,7 +53,9 @@
         'dev': [
             'coverage',
             'flake8',
-            'mitmproxy',  # This should be removed once the mitmproxy backend goes
+            'gunicorn',
+            'httpbin',
+            'mitmproxy',  # Needed by the mitmproxy backend and end2end tests
             'pre-commit',
             'pytest',
             'pytest-cov',

tests/end2end/__init__.py

@@ -0,0 +1,19 @@
+<html>
+    <head>
+        <title>Post JSON to End2End Test</title>
+    </head>
+    <body>
+        <button id="submit" onclick="sendJson()">Submit</button>
+    </body>
+    <script>
+        function sendJson() {
+            let xhr = new XMLHttpRequest();
+            xhr.open("POST", "https://localhost:8085/post", true);
+            xhr.setRequestHeader("Content-Type", "application/json");
+            let data = JSON.stringify(
+                { "hello": "world", "spam": "eggs" }
+            );
+            xhr.send(data);
+        }
+    </script>
+</html>