Move spamhaus DROP/EDROP exclusion to crawl.py

Author: ayeowch

crawl.py

@@ -410,7 +410,7 @@ def list_excluded_networks(txt, networks=None):
         networks = set()
     lines = txt.strip().split("\n")
     for line in lines:
-        line = line.split('#')[0].strip()
+        line = line.split('#')[0].split(';')[0].strip()
         try:
             network = ip_network(unicode(line))
         except ValueError:
@@ -422,11 +422,16 @@ def list_excluded_networks(txt, networks=None):
 
 def update_excluded_networks():
     """
-    Adds bogons into the excluded IPv4 and IPv6 networks.
+    Updates excluded networks with current bogons.
     """
+    CONF['exclude_ipv4_networks'] = CONF['default_exclude_ipv4_networks']
+    CONF['exclude_ipv6_networks'] = CONF['default_exclude_ipv6_networks']
+
     if CONF['exclude_ipv4_bogons']:
         urls = [
             "http://www.team-cymru.org/Services/Bogons/fullbogons-ipv4.txt",
+            "http://www.spamhaus.org/drop/drop.txt",
+            "https://www.spamhaus.org/drop/edrop.txt",
         ]
         for url in urls:
             try:
@@ -498,11 +503,14 @@ def init_conf(argv):
     if exclude_asns:
         CONF['exclude_asns'] = set(exclude_asns.split("\n"))
 
-    CONF['exclude_ipv4_networks'] = list_excluded_networks(
+    CONF['default_exclude_ipv4_networks'] = list_excluded_networks(
         conf.get('crawl', 'exclude_ipv4_networks'))
-    CONF['exclude_ipv6_networks'] = list_excluded_networks(
+    CONF['default_exclude_ipv6_networks'] = list_excluded_networks(
         conf.get('crawl', 'exclude_ipv6_networks'))
 
+    CONF['exclude_ipv4_networks'] = CONF['default_exclude_ipv4_networks']
+    CONF['exclude_ipv6_networks'] = CONF['default_exclude_ipv6_networks']
+
     CONF['exclude_ipv4_bogons'] = conf.getboolean('crawl',
                                                   'exclude_ipv4_bogons')
     CONF['exclude_ipv6_bogons'] = conf.getboolean('crawl',
@@ -561,8 +569,8 @@ def main(argv):
             redis_pipe.delete(key)
         redis_pipe.delete('pending')
         redis_pipe.execute()
-        set_pending()
         update_excluded_networks()
+        set_pending()
         REDIS_CONN.set('crawl:master:state', "running")
 
     # Spawn workers (greenlets) including one worker reserved for cron tasks

seeder.py

@@ -34,12 +34,10 @@
 import operator
 import os
 import random
-import requests
 import sys
 import time
 from collections import defaultdict
 from ConfigParser import ConfigParser
-from ipaddress import ip_address, ip_network
 
 from utils import new_redis_conn
 
@@ -58,16 +56,12 @@ def __init__(self):
         self.nodes = []
         self.addresses = defaultdict(list)
         self.now = 0
-        self.blocklist = set()
-        self.blocklist_timestamp = 0
 
     def export_nodes(self, dump):
         """
         Exports nodes to generate A and AAAA records from the latest snapshot.
         """
         self.now = int(time.time())
-        if self.now - self.blocklist_timestamp > 3600:
-            self.update_blocklist()
         if dump != self.dump:
             try:
                 self.nodes = json.loads(open(dump, "r").read(),
@@ -157,7 +151,6 @@ def filter_nodes(self):
         2) Uptime must be equal or greater than the configured min. age
         3) Max. one node per ASN
         4) Uses default port
-        5) Not listed in blocklist
         """
         consensus_height = self.get_consensus_height()
         min_age = self.get_min_age()
@@ -169,10 +162,7 @@ def filter_nodes(self):
             services = node[5]
             height = node[6]
             asn = node[13]
-            if (port != CONF['port'] or
-                    asn is None or
-                    age < min_age or
-                    self.is_blocked(address)):
+            if port != CONF['port'] or asn is None or age < min_age:
                 continue
             if consensus_height and abs(consensus_height - height) > 2:
                 continue
@@ -205,49 +195,6 @@ def get_min_age(self):
         logging.info("Min. age: %d", min_age)
         return min_age
 
-    def is_blocked(self, address):
-        """
-        Returns True if address is found in blocklist, False if otherwise.
-        """
-        if address.endswith(".onion") or ":" in address:
-            return False
-        for network in self.blocklist:
-            if ip_address(address) in network:
-                logging.debug("Blocked: %s", address)
-                return True
-        return False
-
-    def update_blocklist(self):
-        """
-        Fetches the latest DROP (don't route or peer) list from Spamhaus:
-        http://www.spamhaus.org/faq/section/DROP%20FAQ
-        """
-        urls = [
-            "http://www.spamhaus.org/drop/drop.txt",
-            "http://www.spamhaus.org/drop/edrop.txt",
-        ]
-        self.blocklist.clear()
-        for url in urls:
-            try:
-                response = requests.get(url, timeout=15)
-            except requests.exceptions.RequestException as err:
-                logging.warning(err)
-                continue
-            if response.status_code == 200:
-                for line in response.content.strip().split("\n"):
-                    if line.startswith(";"):
-                        continue
-                    network = line.split(";")[0].strip()
-                    try:
-                        self.blocklist.add(ip_network(unicode(network)))
-                    except ValueError:
-                        continue
-            else:
-                logging.warning("HTTP%d: %s (%s)",
-                                response.status_code, url, response.content)
-        logging.debug("Blocklist entries: %d", len(self.blocklist))
-        self.blocklist_timestamp = self.now
-
 
 def cron():
     """

tests/crawl.py

@@ -0,0 +1 @@
+../crawl.py

tests/test_update_excluded_networks.py

@@ -0,0 +1,20 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+import os
+
+from crawl import CONF, init_conf, update_excluded_networks
+
+
+def test_update_excluded_networks():
+    filepath = os.path.realpath(__file__)
+    confpath = os.path.join(
+        os.path.dirname(filepath), '..', 'conf', 'crawl.conf.default')
+    init_conf([filepath, confpath, 'master'])
+
+    assert len(CONF['default_exclude_ipv4_networks']) == 22
+    assert len(CONF['default_exclude_ipv6_networks']) == 0
+
+    update_excluded_networks()
+
+    assert len(CONF['exclude_ipv4_networks']) > 0
+    assert len(CONF['exclude_ipv6_networks']) == 0