Implement urllib3 native retries

Author: dagwieers

winrm/protocol.py

@@ -26,8 +26,8 @@ class Protocol(object):
     DEFAULT_OPERATION_TIMEOUT_SEC = 20
     DEFAULT_MAX_ENV_SIZE = 153600
     DEFAULT_LOCALE = 'en-US'
-    DEFAULT_RECONNECTION_RETRIES = 5
-    DEFAULT_RECONNECTION_SLEEP = 5
+    DEFAULT_RECONNECTION_RETRIES = 4
+    DEFAULT_RECONNECTION_BACKOFF = 2.0
 
     def __init__(
             self, endpoint, transport='plaintext', username=None,
@@ -42,7 +42,7 @@ def __init__(
             credssp_disable_tlsv1_2=False,
             send_cbt=True,
             reconnection_retries = DEFAULT_RECONNECTION_RETRIES,
-            reconnection_sleep = DEFAULT_RECONNECTION_SLEEP,
+            reconnection_backoff = DEFAULT_RECONNECTION_BACKOFF,
         ):
         """
         @param string endpoint: the WinRM webservice endpoint
@@ -61,8 +61,8 @@ def __init__(
         @param int operation_timeout_sec: maximum allowed time in seconds for any single wsman HTTP operation (default 20). Note that operation timeouts while receiving output (the only wsman operation that should take any significant time, and where these timeouts are expected) will be silently retried indefinitely. # NOQA
         @param string kerberos_hostname_override: the hostname to use for the kerberos exchange (defaults to the hostname in the endpoint URL)
         @param bool message_encryption_enabled: Will encrypt the WinRM messages if set to True and the transport auth supports message encryption (Default True).
-        @param int reconnection_retries: Number of retries on Connection Refused
-        @param int reconnection_sleep: Number of seconds to sleep between reconnection attempts
+        @param int reconnection_retries: Number of retries on connection problems
+        @param float reconnection_backoff: Number of seconds to backoff in between reconnection attempts (first sleeps X, then sleeps 2*X, then sleeps 4*X, ...)
         """
 
         try:
@@ -96,7 +96,7 @@ def __init__(
             credssp_disable_tlsv1_2=credssp_disable_tlsv1_2,
             send_cbt=send_cbt,
             reconnection_retries=reconnection_retries,
-            reconnection_sleep=reconnection_sleep,
+            reconnection_backoff=reconnection_backoff,
         )
 
         self.username = username
@@ -109,7 +109,7 @@ def __init__(
         self.kerberos_hostname_override = kerberos_hostname_override
         self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2
         self.reconnection_retries = reconnection_retries
-        self.reconnection_sleep = reconnection_sleep
+        self.reconnection_backoff = reconnection_backoff
 
     def open_shell(self, i_stream='stdin', o_stream='stdout stderr',
                    working_directory=None, env_vars=None, noprofile=False,

winrm/transport.py

@@ -1,8 +1,7 @@
 from __future__ import unicode_literals
-import inspect
-import os
 import sys
-import time
+import os
+import inspect
 
 is_py2 = sys.version[0] == '2'
 
@@ -64,8 +63,8 @@ def __init__(
             message_encryption='auto',
             credssp_disable_tlsv1_2=False,
             send_cbt=True,
-            reconnection_retries=5,
-            reconnection_sleep=5,
+            reconnection_retries=4,
+            reconnection_backoff=2.0,
         ):
         self.endpoint = endpoint
         self.username = username
@@ -83,7 +82,7 @@ def __init__(
         self.credssp_disable_tlsv1_2 = credssp_disable_tlsv1_2
         self.send_cbt = send_cbt
         self.reconnection_retries = reconnection_retries
-        self.reconnection_sleep = reconnection_sleep
+        self.reconnection_backoff = reconnection_backoff
 
         if self.server_cert_validation not in [None, 'validate', 'ignore']:
             raise WinRMError('invalid server_cert_validation mode: %s' % self.server_cert_validation)
@@ -158,6 +157,15 @@ def build_session(self):
         settings = session.merge_environment_settings(url=self.endpoint, proxies={}, stream=None,
                                                       verify=None, cert=None)
 
+        # Retry on connection errors, with a backoff factor
+        retries = requests.packages.urllib3.util.retry.Retry(total=self.reconnection_retries,
+                                                             connect=self.reconnection_retries,
+                                                             read=self.reconnection_retries,
+                                                             backoff_factor=self.reconnection_backoff,
+                                                             status_forcelist=[425, 429, 503])
+        session.mount('http://', requests.adapters.HTTPAdapter(max_retries=retries))
+        session.mount('https://', requests.adapters.HTTPAdapter(max_retries=retries))
+
         # get proxy settings from env
         # FUTURE: allow proxy to be passed in directly to supersede this value
         session.proxies = settings['proxies']
@@ -266,20 +274,7 @@ def send_message(self, message):
 
     def _send_message_request(self, prepared_request, message):
         try:
-
-            # Retry connection on 'Connection refused'
-            for attempt in range(self.reconnection_retries):
-                try:
-                    response = self.session.send(prepared_request, timeout=self.read_timeout_sec)
-                except requests.packages.urllib3.exceptions.NewConnectionError as e:
-                    time.sleep(self.reconnection_sleep)
-#                except requests.exceptions.ConnectionError as e:
-#                    if attempt == 4 or 'connection refused' not in str(e).lower():
-#                        raise
-#                    time.sleep(self.reconnection_sleep)
-                else:
-                    break
-
+            response = self.session.send(prepared_request, timeout=self.read_timeout_sec)
             response.raise_for_status()
             return response
         except requests.HTTPError as ex: