Added proper logging, verbosity levels, and other refactoring

Thanks to maxme for the help on this!

Author: jordan-wright

dumpmon.py

@@ -17,15 +17,25 @@
 import twitter
 from settings import CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET
 import threading
+import logging
 
 
 def monitor():
     '''
     monitor() - Main function... creates and starts threads
 
     '''
-    log('[*] Monitoring...')
-    log('[*] Ctrl+C to quit')
+    import argparse
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "-v", "--verbose", help="more verbose", action="store_true")
+    args = parser.parse_args()
+    level = logging.INFO
+    if args.verbose:
+        level = logging.DEBUG
+    logging.basicConfig(
+        format='%(asctime)s [%(levelname)s] %(message)s', level=level)
+    logging.info('Monitoring...')
     bot = twitter.Api(consumer_key=CONSUMER_KEY,
                       consumer_secret=CONSUMER_SECRET,
                       access_token_key=ACCESS_TOKEN,
@@ -35,11 +45,11 @@ def monitor():
     tweet_lock = threading.Lock()
 
     pastebin_thread = threading.Thread(
-        target=Pastebin().monitor, args=[bot, log_lock, tweet_lock])
+        target=Pastebin().monitor, args=[bot, tweet_lock])
     slexy_thread = threading.Thread(
-        target=Slexy().monitor, args=[bot, log_lock, tweet_lock])
+        target=Slexy().monitor, args=[bot, tweet_lock])
     pastie_thead = threading.Thread(
-        target=Pastie().monitor, args=[bot, log_lock, tweet_lock])
+        target=Pastie().monitor, args=[bot, tweet_lock])
 
     for thread in (pastebin_thread, slexy_thread, pastie_thead):
         thread.daemon = True
@@ -50,7 +60,7 @@ def monitor():
         while(1):
             sleep(5)
     except KeyboardInterrupt:
-        log('Stopped.')
+        logging.warn('Stopped.')
 
 
 if __name__ == "__main__":

lib/Paste.py

@@ -1,5 +1,6 @@
 from .regexes import regexes
 import settings
+import logging
 
 
 def log(text):
@@ -47,12 +48,12 @@ def match(self):
         self.num_hashes = len(self.hashes)
         for regex in regexes['db_keywords']:
             if regex.search(self.text):
-                log('\t[+] ' + regex.search(self.text).group(1))
+                logging.debug('\t[+] ' + regex.search(self.text).group(1))
                 self.db_keywords += round(1/float(
                     len(regexes['db_keywords'])), 2)
         for regex in regexes['blacklist']:
             if regex.search(self.text):
-                log('\t[-] ' + regex.search(self.text).group(1))
+                logging.debug('\t[-] ' + regex.search(self.text).group(1))
                 self.db_keywords -= round(1.25 * (
                     1/float(len(regexes['db_keywords']))), 2)
         if (self.num_emails >= settings.EMAIL_THRESHOLD) or (self.num_hashes >= settings.HASH_THRESHOLD) or (self.db_keywords >= settings.DB_KEYWORDS_THRESHOLD):

lib/Pastebin.py

@@ -5,6 +5,7 @@
 from time import sleep
 from settings import SLEEP_PASTEBIN
 from twitter import TwitterError
+import logging
 
 
 class PastebinPaste(Paste):
@@ -21,11 +22,12 @@ def __init__(self, last_id=None):
             last_id = None
         self.ref_id = last_id
         self.BASE_URL = 'http://pastebin.com'
+        self.sleep = SLEEP_PASTEBIN
         super(Pastebin, self).__init__()
 
     def update(self):
         '''update(self) - Fill Queue with new Pastebin IDs'''
-        print('[*] Retrieving Pastebin ID\'s')
+        logging.info('Retrieving Pastebin ID\'s')
         results = BeautifulSoup(helper.download(self.BASE_URL + '/archive')).find_all(
             lambda tag: tag.name == 'td' and tag.a and '/archive/' not in tag.a['href'] and tag.a['href'][1:])
         new_pastes = []
@@ -38,32 +40,7 @@ def update(self):
                 break
             new_pastes.append(paste)
         for entry in new_pastes[::-1]:
-            print('[+] Adding URL: ' + entry.url)
+            logging.info('Adding URL: ' + entry.url)
             self.put(entry)
-
-    def monitor(self, bot, l_lock, t_lock):
-        self.update()
-        while(1):
-            while not self.empty():
-                paste = self.get()
-                self.ref_id = paste.id
-                with l_lock:
-                    helper.log('[*] Checking ' + paste.url)
-                paste.text = helper.download(paste.url)
-                with l_lock:
-                    tweet = helper.build_tweet(paste)
-                if tweet:
-                    print(tweet)
-                    with t_lock:
-                        helper.record(tweet)
-                        try:
-                            bot.PostUpdate(tweet)
-                        except TwitterError:
-                            pass
-            self.update()
-            # If no new results... sleep for 5 sec
-            while self.empty():
-                with l_lock:
-                    helper.log('[*] No results... sleeping')
-                sleep(SLEEP_PASTEBIN)
-                self.update()
+    def get_paste_text(self, paste):
+        return helper.download(paste.url)

lib/Pastie.py

@@ -5,6 +5,7 @@
 from time import sleep
 from settings import SLEEP_PASTIE
 from twitter import TwitterError
+import logging
 
 
 class PastiePaste(Paste):
@@ -21,11 +22,12 @@ def __init__(self, last_id=None):
             last_id = None
         self.ref_id = last_id
         self.BASE_URL = 'http://pastie.org'
+        self.sleep = SLEEP_PASTIE
         super(Pastie, self).__init__()
 
     def update(self):
         '''update(self) - Fill Queue with new Pastie IDs'''
-        print('[*] Retrieving Pastie ID\'s')
+        logging.info('Retrieving Pastie ID\'s')
         results = [tag for tag in BeautifulSoup(helper.download(
             self.BASE_URL + '/pastes')).find_all('p', 'link') if tag.a]
         new_pastes = []
@@ -39,34 +41,8 @@ def update(self):
                 break
             new_pastes.append(paste)
         for entry in new_pastes[::-1]:
-            print('[+] Adding URL: ' + entry.url)
+            logging.debug('Adding URL: ' + entry.url)
             self.put(entry)
 
-    def monitor(self, bot, l_lock, t_lock):
-        self.update()
-        while(1):
-            while not self.empty():
-                paste = self.get()
-                self.ref_id = paste.id
-                with l_lock:
-                    helper.log('[*] Checking ' + paste.url)
-                # goober pastie - Not actually showing *raw* text.. Still need
-                # to parse it out
-                paste.text = BeautifulSoup(helper.download(paste.url)).pre.text
-                with l_lock:
-                    tweet = helper.build_tweet(paste)
-                if tweet:
-                    print(tweet)
-                    with t_lock:
-                        helper.record(tweet)
-                        try:
-                            bot.PostUpdate(tweet)
-                        except TwitterError:
-                            pass
-            self.update()
-            # If no new results... sleep for 5 sec
-            while self.empty():
-                with l_lock:
-                    helper.log('[*] No results... sleeping')
-                sleep(SLEEP_PASTIE)
-                self.update()
+    def get_paste_text(self, paste):
+        return BeautifulSoup(helper.download(paste.url)).pre.text

lib/Site.py

@@ -1,12 +1,17 @@
-from queue import Queue
+from Queue import Queue
 import requests
 import time
 from requests import ConnectionError
+import logging
+import helper
 
 
 class Site(object):
     '''
-    Site - parent class used for a generic 'Queue' structure with a few helper methods and features. Impelements the following methods:
+    Site - parent class used for a generic
+    'Queue' structure with a few helper methods
+    and features. Implements the following methods:
+
             empty() - Is the Queue empty
             get(): Get the next item in the queue
             put(item): Puts an item in the queue
@@ -53,3 +58,28 @@ def clear(self):
 
     def list(self):
         print('\n'.join(url for url in self.queue))
+
+    def monitor(self, bot, t_lock):
+        self.update()
+        while(1):
+            while not self.empty():
+                paste = self.get()
+                self.ref_id = paste.id
+                logging.info('[*] Checking ' + paste.url)
+                # goober pastie - Not actually showing *raw* text.. Still need
+                # to parse it out
+                paste.text = self.get_paste_text(paste)
+                tweet = helper.build_tweet(paste)
+                if tweet:
+                    logging.info(tweet)
+                    with t_lock:
+                        helper.record(tweet)
+                        try:
+                            bot.PostUpdate(tweet)
+                        except TwitterError:
+                            pass
+            self.update()
+            while self.empty():
+                logging.debug('[*] No results... sleeping')
+                time.sleep(self.sleep)
+                self.update()

lib/Slexy.py

@@ -5,6 +5,7 @@
 from time import sleep
 from settings import SLEEP_SLEXY
 from twitter import TwitterError
+import logging
 
 
 class SlexyPaste(Paste):
@@ -21,11 +22,12 @@ def __init__(self, last_id=None):
             last_id = None
         self.ref_id = last_id
         self.BASE_URL = 'http://slexy.org'
+        self.sleep = SLEEP_SLEXY
         super(Slexy, self).__init__()
 
     def update(self):
         '''update(self) - Fill Queue with new Slexy IDs'''
-        print('[*] Retrieving Slexy ID\'s')
+        logging.info('[*] Retrieving Slexy ID\'s')
         results = BeautifulSoup(helper.download(self.BASE_URL + '/recent')).find_all(
             lambda tag: tag.name == 'td' and tag.a and '/view/' in tag.a['href'])
         new_pastes = []
@@ -38,32 +40,8 @@ def update(self):
                 break
             new_pastes.append(paste)
         for entry in new_pastes[::-1]:
-            print('[+] Adding URL: ' + entry.url)
+            logging.info('[+] Adding URL: ' + entry.url)
             self.put(entry)
 
-    def monitor(self, bot, l_lock, t_lock):
-        self.update()
-        while(1):
-            while not self.empty():
-                paste = self.get()
-                self.ref_id = paste.id
-                with l_lock:
-                    helper.log('[*] Checking ' + paste.url)
-                paste.text = helper.download(paste.url)
-                with l_lock:
-                    tweet = helper.build_tweet(paste)
-                if tweet:
-                    print(tweet)
-                    with t_lock:
-                        helper.record(tweet)
-                        try:
-                            bot.PostUpdate(tweet)
-                        except TwitterError:
-                            pass
-            self.update()
-            # If no new results... sleep for 5 sec
-            while self.empty():
-                with l_lock:
-                    helper.log('[*] No results... sleeping')
-                sleep(SLEEP_SLEXY)
-                self.update()
+    def get_paste_text(self, paste):
+        return helper.download(paste.url, paste.headers)

lib/helper.py

@@ -7,6 +7,7 @@
 import requests
 import settings
 from time import sleep, strftime
+import logging
 
 
 r = requests.Session()