Skip to content

Commit 3adcb5e

Browse files
committed
Added proper logging, verbosity levels, and other refactoring
Thanks to maxme for the help on this!
1 parent 2dbbad7 commit 3adcb5e

File tree

7 files changed

+70
-97
lines changed

7 files changed

+70
-97
lines changed

dumpmon.py

+16-6
Original file line numberDiff line numberDiff line change
@@ -17,15 +17,25 @@
1717
import twitter
1818
from settings import CONSUMER_KEY, CONSUMER_SECRET, ACCESS_TOKEN, ACCESS_TOKEN_SECRET
1919
import threading
20+
import logging
2021

2122

2223
def monitor():
2324
'''
2425
monitor() - Main function... creates and starts threads
2526
2627
'''
27-
log('[*] Monitoring...')
28-
log('[*] Ctrl+C to quit')
28+
import argparse
29+
parser = argparse.ArgumentParser()
30+
parser.add_argument(
31+
"-v", "--verbose", help="more verbose", action="store_true")
32+
args = parser.parse_args()
33+
level = logging.INFO
34+
if args.verbose:
35+
level = logging.DEBUG
36+
logging.basicConfig(
37+
format='%(asctime)s [%(levelname)s] %(message)s', level=level)
38+
logging.info('Monitoring...')
2939
bot = twitter.Api(consumer_key=CONSUMER_KEY,
3040
consumer_secret=CONSUMER_SECRET,
3141
access_token_key=ACCESS_TOKEN,
@@ -35,11 +45,11 @@ def monitor():
3545
tweet_lock = threading.Lock()
3646

3747
pastebin_thread = threading.Thread(
38-
target=Pastebin().monitor, args=[bot, log_lock, tweet_lock])
48+
target=Pastebin().monitor, args=[bot, tweet_lock])
3949
slexy_thread = threading.Thread(
40-
target=Slexy().monitor, args=[bot, log_lock, tweet_lock])
50+
target=Slexy().monitor, args=[bot, tweet_lock])
4151
pastie_thead = threading.Thread(
42-
target=Pastie().monitor, args=[bot, log_lock, tweet_lock])
52+
target=Pastie().monitor, args=[bot, tweet_lock])
4353

4454
for thread in (pastebin_thread, slexy_thread, pastie_thead):
4555
thread.daemon = True
@@ -50,7 +60,7 @@ def monitor():
5060
while(1):
5161
sleep(5)
5262
except KeyboardInterrupt:
53-
log('Stopped.')
63+
logging.warn('Stopped.')
5464

5565

5666
if __name__ == "__main__":

lib/Paste.py

+3-2
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
from .regexes import regexes
22
import settings
3+
import logging
34

45

56
def log(text):
@@ -47,12 +48,12 @@ def match(self):
4748
self.num_hashes = len(self.hashes)
4849
for regex in regexes['db_keywords']:
4950
if regex.search(self.text):
50-
log('\t[+] ' + regex.search(self.text).group(1))
51+
logging.debug('\t[+] ' + regex.search(self.text).group(1))
5152
self.db_keywords += round(1/float(
5253
len(regexes['db_keywords'])), 2)
5354
for regex in regexes['blacklist']:
5455
if regex.search(self.text):
55-
log('\t[-] ' + regex.search(self.text).group(1))
56+
logging.debug('\t[-] ' + regex.search(self.text).group(1))
5657
self.db_keywords -= round(1.25 * (
5758
1/float(len(regexes['db_keywords']))), 2)
5859
if (self.num_emails >= settings.EMAIL_THRESHOLD) or (self.num_hashes >= settings.HASH_THRESHOLD) or (self.db_keywords >= settings.DB_KEYWORDS_THRESHOLD):

lib/Pastebin.py

+6-29
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from time import sleep
66
from settings import SLEEP_PASTEBIN
77
from twitter import TwitterError
8+
import logging
89

910

1011
class PastebinPaste(Paste):
@@ -21,11 +22,12 @@ def __init__(self, last_id=None):
2122
last_id = None
2223
self.ref_id = last_id
2324
self.BASE_URL = 'http://pastebin.com'
25+
self.sleep = SLEEP_PASTEBIN
2426
super(Pastebin, self).__init__()
2527

2628
def update(self):
2729
'''update(self) - Fill Queue with new Pastebin IDs'''
28-
print('[*] Retrieving Pastebin ID\'s')
30+
logging.info('Retrieving Pastebin ID\'s')
2931
results = BeautifulSoup(helper.download(self.BASE_URL + '/archive')).find_all(
3032
lambda tag: tag.name == 'td' and tag.a and '/archive/' not in tag.a['href'] and tag.a['href'][1:])
3133
new_pastes = []
@@ -38,32 +40,7 @@ def update(self):
3840
break
3941
new_pastes.append(paste)
4042
for entry in new_pastes[::-1]:
41-
print('[+] Adding URL: ' + entry.url)
43+
logging.info('Adding URL: ' + entry.url)
4244
self.put(entry)
43-
44-
def monitor(self, bot, l_lock, t_lock):
45-
self.update()
46-
while(1):
47-
while not self.empty():
48-
paste = self.get()
49-
self.ref_id = paste.id
50-
with l_lock:
51-
helper.log('[*] Checking ' + paste.url)
52-
paste.text = helper.download(paste.url)
53-
with l_lock:
54-
tweet = helper.build_tweet(paste)
55-
if tweet:
56-
print(tweet)
57-
with t_lock:
58-
helper.record(tweet)
59-
try:
60-
bot.PostUpdate(tweet)
61-
except TwitterError:
62-
pass
63-
self.update()
64-
# If no new results... sleep for 5 sec
65-
while self.empty():
66-
with l_lock:
67-
helper.log('[*] No results... sleeping')
68-
sleep(SLEEP_PASTEBIN)
69-
self.update()
45+
def get_paste_text(self, paste):
46+
return helper.download(paste.url)

lib/Pastie.py

+6-30
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from time import sleep
66
from settings import SLEEP_PASTIE
77
from twitter import TwitterError
8+
import logging
89

910

1011
class PastiePaste(Paste):
@@ -21,11 +22,12 @@ def __init__(self, last_id=None):
2122
last_id = None
2223
self.ref_id = last_id
2324
self.BASE_URL = 'http://pastie.org'
25+
self.sleep = SLEEP_PASTIE
2426
super(Pastie, self).__init__()
2527

2628
def update(self):
2729
'''update(self) - Fill Queue with new Pastie IDs'''
28-
print('[*] Retrieving Pastie ID\'s')
30+
logging.info('Retrieving Pastie ID\'s')
2931
results = [tag for tag in BeautifulSoup(helper.download(
3032
self.BASE_URL + '/pastes')).find_all('p', 'link') if tag.a]
3133
new_pastes = []
@@ -39,34 +41,8 @@ def update(self):
3941
break
4042
new_pastes.append(paste)
4143
for entry in new_pastes[::-1]:
42-
print('[+] Adding URL: ' + entry.url)
44+
logging.debug('Adding URL: ' + entry.url)
4345
self.put(entry)
4446

45-
def monitor(self, bot, l_lock, t_lock):
46-
self.update()
47-
while(1):
48-
while not self.empty():
49-
paste = self.get()
50-
self.ref_id = paste.id
51-
with l_lock:
52-
helper.log('[*] Checking ' + paste.url)
53-
# goober pastie - Not actually showing *raw* text.. Still need
54-
# to parse it out
55-
paste.text = BeautifulSoup(helper.download(paste.url)).pre.text
56-
with l_lock:
57-
tweet = helper.build_tweet(paste)
58-
if tweet:
59-
print(tweet)
60-
with t_lock:
61-
helper.record(tweet)
62-
try:
63-
bot.PostUpdate(tweet)
64-
except TwitterError:
65-
pass
66-
self.update()
67-
# If no new results... sleep for 5 sec
68-
while self.empty():
69-
with l_lock:
70-
helper.log('[*] No results... sleeping')
71-
sleep(SLEEP_PASTIE)
72-
self.update()
47+
def get_paste_text(self, paste):
48+
return BeautifulSoup(helper.download(paste.url)).pre.text

lib/Site.py

+32-2
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,17 @@
1-
from queue import Queue
1+
from Queue import Queue
22
import requests
33
import time
44
from requests import ConnectionError
5+
import logging
6+
import helper
57

68

79
class Site(object):
810
'''
9-
Site - parent class used for a generic 'Queue' structure with a few helper methods and features. Impelements the following methods:
11+
Site - parent class used for a generic
12+
'Queue' structure with a few helper methods
13+
and features. Implements the following methods:
14+
1015
empty() - Is the Queue empty
1116
get(): Get the next item in the queue
1217
put(item): Puts an item in the queue
@@ -53,3 +58,28 @@ def clear(self):
5358

5459
def list(self):
5560
print('\n'.join(url for url in self.queue))
61+
62+
def monitor(self, bot, t_lock):
63+
self.update()
64+
while(1):
65+
while not self.empty():
66+
paste = self.get()
67+
self.ref_id = paste.id
68+
logging.info('[*] Checking ' + paste.url)
69+
# goober pastie - Not actually showing *raw* text.. Still need
70+
# to parse it out
71+
paste.text = self.get_paste_text(paste)
72+
tweet = helper.build_tweet(paste)
73+
if tweet:
74+
logging.info(tweet)
75+
with t_lock:
76+
helper.record(tweet)
77+
try:
78+
bot.PostUpdate(tweet)
79+
except TwitterError:
80+
pass
81+
self.update()
82+
while self.empty():
83+
logging.debug('[*] No results... sleeping')
84+
time.sleep(self.sleep)
85+
self.update()

lib/Slexy.py

+6-28
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
from time import sleep
66
from settings import SLEEP_SLEXY
77
from twitter import TwitterError
8+
import logging
89

910

1011
class SlexyPaste(Paste):
@@ -21,11 +22,12 @@ def __init__(self, last_id=None):
2122
last_id = None
2223
self.ref_id = last_id
2324
self.BASE_URL = 'http://slexy.org'
25+
self.sleep = SLEEP_SLEXY
2426
super(Slexy, self).__init__()
2527

2628
def update(self):
2729
'''update(self) - Fill Queue with new Slexy IDs'''
28-
print('[*] Retrieving Slexy ID\'s')
30+
logging.info('[*] Retrieving Slexy ID\'s')
2931
results = BeautifulSoup(helper.download(self.BASE_URL + '/recent')).find_all(
3032
lambda tag: tag.name == 'td' and tag.a and '/view/' in tag.a['href'])
3133
new_pastes = []
@@ -38,32 +40,8 @@ def update(self):
3840
break
3941
new_pastes.append(paste)
4042
for entry in new_pastes[::-1]:
41-
print('[+] Adding URL: ' + entry.url)
43+
logging.info('[+] Adding URL: ' + entry.url)
4244
self.put(entry)
4345

44-
def monitor(self, bot, l_lock, t_lock):
45-
self.update()
46-
while(1):
47-
while not self.empty():
48-
paste = self.get()
49-
self.ref_id = paste.id
50-
with l_lock:
51-
helper.log('[*] Checking ' + paste.url)
52-
paste.text = helper.download(paste.url)
53-
with l_lock:
54-
tweet = helper.build_tweet(paste)
55-
if tweet:
56-
print(tweet)
57-
with t_lock:
58-
helper.record(tweet)
59-
try:
60-
bot.PostUpdate(tweet)
61-
except TwitterError:
62-
pass
63-
self.update()
64-
# If no new results... sleep for 5 sec
65-
while self.empty():
66-
with l_lock:
67-
helper.log('[*] No results... sleeping')
68-
sleep(SLEEP_SLEXY)
69-
self.update()
46+
def get_paste_text(self, paste):
47+
return helper.download(paste.url, paste.headers)

lib/helper.py

+1
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
import requests
88
import settings
99
from time import sleep, strftime
10+
import logging
1011

1112

1213
r = requests.Session()

0 commit comments

Comments
 (0)