Make exp backoff a iterator, add configurability

Author: hcoura

scrapy_crawlera/middleware.py

@@ -10,7 +10,7 @@
 from scrapy.exceptions import ScrapyDeprecationWarning
 from twisted.internet.error import ConnectionRefusedError, ConnectionDone
 
-from scrapy_crawlera.utils import exp_backoff_full_jitter
+from scrapy_crawlera.utils import exp_backoff
 
 
 class CrawleraMiddleware(object):
@@ -24,9 +24,9 @@ class CrawleraMiddleware(object):
     preserve_delay = False
     header_prefix = 'X-Crawlera-'
     conflicting_headers = ('X-Crawlera-Profile', 'X-Crawlera-UA')
-    noslaves_attempts = 0
-    noslaves_base_delay = 15
-    noslaves_max_delay = 180
+    backoff_step = 15
+    backoff_max = 180
+    exp_backoff = None
 
     _settings = [
         ('apikey', str),
@@ -36,6 +36,8 @@ class CrawleraMiddleware(object):
         ('maxbans', int),
         ('download_timeout', int),
         ('preserve_delay', bool),
+        ('backoff_step', int),
+        ('backoff_max', int),
     ]
 
     def __init__(self, crawler):
@@ -71,6 +73,7 @@ def open_spider(self, spider):
                 "To avoid this behaviour you can use the CRAWLERA_PRESERVE_DELAY setting but keep in mind that this may slow down the crawl significantly")
 
         self._headers = self.crawler.settings.get('CRAWLERA_DEFAULT_HEADERS', {}).items()
+        self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
 
     def _settings_get(self, type_, *a, **kw):
         if type_ is int:
@@ -158,8 +161,7 @@ def process_response(self, request, response, spider):
         self._restore_original_delay(request)
 
         if self._is_no_available_proxies(response):
-            after = self._get_noslaves_delay()
-            self._set_custom_delay(request, after)
+            self._set_custom_delay(request, next(self.exp_backoff))
         else:
             self._reset_noslaves_delay()
 
@@ -207,22 +209,9 @@ def _get_slot(self, request):
         key = self._get_slot_key(request)
         return key, self.crawler.engine.downloader.slots.get(key)
 
-    def _get_noslaves_delay(self):
-        """
-        Returns the amount of delay to use in case of no available proxies,
-        also increments the number of attempts due to no proxies
-        """
-        delay = exp_backoff_full_jitter(
-            self.noslaves_attempts,
-            self.noslaves_max_delay,
-            self.noslaves_base_delay
-        )
-        self.noslaves_attempts += 1
-        return delay
-
     def _reset_noslaves_delay(self):
         """Reset the number of attempts due to no available proxies"""
-        self.noslaves_attempts = 0
+        self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
 
     def _set_custom_delay(self, request, delay):
         """Set custom delay for slot and save original one."""

scrapy_crawlera/utils.py

@@ -1,17 +1,16 @@
 import math
 import random
 
-
-def exp_backoff(attempt, cap, base):
-    """ Exponential backoff time """
-    # this is a numerically stable version of
-    # min(cap, base * 2 ** attempt)
-    max_attempts = math.log(cap / base, 2)
-    if attempt <= max_attempts:
-        return base * 2 ** attempt
-    return cap
+from itertools import count
 
 
-def exp_backoff_full_jitter(attempt, cap, base):
+def exp_backoff(step, max):
     """ Exponential backoff time with Full Jitter """
-    return random.uniform(0, exp_backoff(attempt, cap, base))
+    # this is a numerically stable version of
+    # min(max, step * 2 ** attempt)
+    max_attempts = math.log(max / step, 2)
+    for attempt in count(0, 1):
+        if attempt < max_attempts:
+            yield random.uniform(0, step * 2 ** attempt)
+        else:
+            yield random.uniform(0, max)

tests/test_crawlera.py

@@ -14,6 +14,8 @@
 from scrapy_crawlera import CrawleraMiddleware
 import os
 
+from scrapy_crawlera.utils import exp_backoff
+
 
 class MockedSlot(object):
 
@@ -478,9 +480,12 @@ def test_noslaves_delays(self, random_uniform_patch):
         url = 'http://www.scrapytest.org'
         ban_url = 'http://ban.me'
         max_delay = 70
-        initial_delay = 15
+        backoff_step = 15
         default_delay = 0
 
+        self.settings['CRAWLERA_BACKOFF_STEP'] = backoff_step
+        self.settings['CRAWLERA_BACKOFF_MAX'] = max_delay
+
         self.spider.crawlera_enabled = True
         crawler = self._mock_crawler(self.spider, self.settings)
         mw = self.mwcls.from_crawler(crawler)
@@ -497,13 +502,12 @@ def test_noslaves_delays(self, random_uniform_patch):
 
         # delays grow exponentially
         mw.process_response(noslaves_req, noslaves_res, self.spider)
-        self.assertEqual(slot.delay, initial_delay)
-
+        self.assertEqual(slot.delay, backoff_step)
         mw.process_response(noslaves_req, noslaves_res, self.spider)
-        self.assertEqual(slot.delay, initial_delay * 2 ** 1)
+        self.assertEqual(slot.delay, backoff_step * 2 ** 1)
 
         mw.process_response(noslaves_req, noslaves_res, self.spider)
-        self.assertEqual(slot.delay, initial_delay * 2 ** 2)
+        self.assertEqual(slot.delay, backoff_step * 2 ** 2)
 
         mw.process_response(noslaves_req, noslaves_res, self.spider)
         self.assertEqual(slot.delay, max_delay)
@@ -517,7 +521,7 @@ def test_noslaves_delays(self, random_uniform_patch):
         self.assertEqual(slot.delay, default_delay)
 
         mw.process_response(noslaves_req, noslaves_res, self.spider)
-        self.assertEqual(slot.delay, initial_delay)
+        self.assertEqual(slot.delay, backoff_step)
 
         good_req = Request(url, meta={'download_slot': slot_key})
         good_res = Response(