Merge pull request #75 from scrapy-plugins/crawlera-on-demand

Enable Crawlera On Demand

Author: hcoura

docs/settings.rst

@@ -63,3 +63,15 @@ CRAWLERA_BACKOFF_MAX
 Default: ``180``
 
 Max value for exponential backoff as showed in the formula above.
+
+CRAWLERA_FORCE_ENABLE_ON_HTTP_CODES
+------------------------------------
+
+Default: ``[]``
+
+List of HTTP response status codes that warrant enabling Crawlera for the
+corresponding domain.
+
+When a response with one of these HTTP status codes is received after a request
+that did not go through Crawlera, the request is retried with Crawlera, and any
+new request to the same domain is also sent through Crawlera.

scrapy_crawlera/middleware.py

@@ -27,7 +27,9 @@ class CrawleraMiddleware(object):
     backoff_step = 15
     backoff_max = 180
     exp_backoff = None
+    force_enable_on_http_codes = []
     max_auth_retry_times = 10
+    enabled_for_domain = {}
 
     _settings = [
         ('apikey', str),
@@ -37,6 +39,7 @@ class CrawleraMiddleware(object):
         ('preserve_delay', bool),
         ('backoff_step', int),
         ('backoff_max', int),
+        ('force_enable_on_http_codes', list),
     ]
 
     def __init__(self, crawler):
@@ -54,15 +57,23 @@ def from_crawler(cls, crawler):
 
     def open_spider(self, spider):
         self.enabled = self.is_enabled(spider)
-        if not self.enabled:
-            return
-
         self.spider = spider
 
         for k, type_ in self._settings:
             setattr(self, k, self._get_setting_value(spider, k, type_))
 
+        self._headers = self.crawler.settings.get('CRAWLERA_DEFAULT_HEADERS', {}).items()
+        self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
+
+        if not self.enabled and not self.force_enable_on_http_codes:
+            return
+
+        if not self.apikey:
+            logging.warning("Crawlera can't be used without a APIKEY", extra={'spider': spider})
+            return
+
         self._proxyauth = self.get_proxyauth(spider)
+
         logging.info(
             "Using crawlera at %s (apikey: %s)" % (self.url, self.apikey[:7]),
             extra={'spider': spider},
@@ -77,9 +88,6 @@ def open_spider(self, spider):
                 extra={'spider': spider},
             )
 
-        self._headers = self.crawler.settings.get('CRAWLERA_DEFAULT_HEADERS', {}).items()
-        self.exp_backoff = exp_backoff(self.backoff_step, self.backoff_max)
-
     def _settings_get(self, type_, *a, **kw):
         if type_ is int:
             return self.crawler.settings.getint(*a, **kw)
@@ -165,7 +173,11 @@ def _is_auth_error(self, response):
 
     def process_response(self, request, response, spider):
         if not self._is_enabled_for_request(request):
-            return response
+            return self._handle_not_enabled_response(request, response)
+
+        if not self._is_crawlera_response(response):
+            return request
+
         key = self._get_slot_key(request)
         self._restore_original_delay(request)
 
@@ -216,6 +228,13 @@ def process_exception(self, request, exception, spider):
             self._clear_dns_cache()
             self._set_custom_delay(request, self.connection_refused_delay)
 
+    def _handle_not_enabled_response(self, request, response):
+        if self._should_enable_for_response(response):
+            domain = self._get_url_domain(request.url)
+            self.enabled_for_domain[domain] = True
+            return request
+        return response
+
     def _retry_auth(self, response, request, spider):
         logging.warning(
             "Retrying crawlera request for authentication issue",
@@ -232,8 +251,21 @@ def _clear_dns_cache(self):
         # so client can reconnect trough DNS failover.
         dnscache.pop(urlparse(self.url).hostname, None)
 
+    def _should_enable_for_response(self, response):
+        return response.status in self.force_enable_on_http_codes
+
     def _is_enabled_for_request(self, request):
-        return self.enabled and not request.meta.get('dont_proxy', False)
+        domain = self._get_url_domain(request.url)
+        domain_enabled = self.enabled_for_domain.get(domain, False)
+        dont_proxy = request.meta.get('dont_proxy', False)
+        return (domain_enabled or self.enabled) and not dont_proxy
+
+    def _get_url_domain(self, url):
+        parsed = urlparse(url)
+        return parsed.netloc
+
+    def _is_crawlera_response(self, response):
+        return bool(response.headers.get("X-Crawlera-Version"))
 
     def _get_slot_key(self, request):
         return request.meta.get('download_slot')