Implemented Multithreading

Bot now posts to Twitter (!)
Cleaned up imports
Worked on regexes

Author: jordan-wright

dumpmon.py

@@ -11,6 +11,7 @@
 from lib.regexes import regexes
 from lib.Pastebin import Pastebin, PastebinPaste
 from lib.Slexy import Slexy, SlexyPaste
+from lib.Pastie import Pastie, PastiePaste
 from lib.helper import log
 from time import sleep
 import twitter
@@ -33,12 +34,12 @@ def monitor():
 	tweet_lock = threading.Lock()
 
 	pastebin_thread = threading.Thread(target=Pastebin().monitor, args=[bot,log_lock, tweet_lock])
-	pastebin_thread.daemon = True
-	pastebin_thread.start()
-
 	slexy_thread = threading.Thread(target=Slexy().monitor, args=[bot,log_lock, tweet_lock])
-	slexy_thread.daemon = True
-	slexy_thread.start()
+	pastie_thead = threading.Thread(target=Pastie().monitor, args=[bot,log_lock, tweet_lock])
+
+	for thread in (pastebin_thread, slexy_thread, pastie_thead):
+		thread.daemon = True
+		thread.start()
 
 	# Let threads run
 	try:

lib/Paste.py

@@ -45,15 +45,16 @@ def match(self):
 		self.num_hashes = len(self.hashes)
 		for regex in regexes['db_keywords']:
 			if regex.search(self.text):
-				log(regex.search(self.text).group(1))
+				log('\t[+] ' + regex.search(self.text).group(1))
 				self.db_keywords += round(1/float(len(regexes['db_keywords'])), 2)
 		for regex in regexes['blacklist']:
 			if regex.search(self.text):
-				log(regex.search(self.text).group(1))
+				log('\t[-] ' + regex.search(self.text).group(1))
 				self.db_keywords -= round(1.25 * (1/float(len(regexes['db_keywords']))), 2)
 		if (self.num_emails >= settings.EMAIL_THRESHOLD) or (self.num_hashes >= settings.HASH_THRESHOLD) or (self.db_keywords >= settings.DB_KEYWORDS_THRESHOLD):
 			self.type = 'db_dump'
 		if regexes['cisco_hash'].search(self.text) or regexes['cisco_pass'].search(self.text): self.type = 'Cisco'
+		if regexes['honeypot'].search(self.text): self.type = 'honeypot'
 		if regexes['google_api'].search(self.text): self.type = 'google_api'
 		#if regexes['juniper'].search(self.text): self.type = 'Juniper'
 		return self.type

lib/Pastebin.py

@@ -40,19 +40,19 @@ def monitor(self, bot, l_lock, t_lock):
 				paste = self.get()
 				self.ref_id = paste.id
 				with l_lock:
-					helper.log('Checking ' + paste.url)
+					helper.log('[*] Checking ' + paste.url)
 				paste.text = helper.download(paste.url)
 				with l_lock:
 					tweet = helper.build_tweet(paste)
 				if tweet:
 					print tweet
 					with t_lock:
 						helper.record(tweet)
-						#bot.PostUpdate(paste.url, tweet)
+						bot.PostUpdate(tweet)
 			self.update()
 			# If no new results... sleep for 5 sec
 			while self.empty():
 				with l_lock:
-					helper.log('No results... sleeping')
+					helper.log('[*] No results... sleeping')
 				sleep(SLEEP_PASTEBIN)
 				self.update()

lib/Pastie.py

@@ -3,7 +3,7 @@
 from bs4 import BeautifulSoup
 import helper
 from time import sleep
-from settings import SLEEP_SLEXY
+from settings import SLEEP_PASTIE
 
 class PastiePaste(Paste):
 	def __init__(self, id):
@@ -19,8 +19,8 @@ def __init__(self, last_id=None):
 		self.BASE_URL = 'http://pastie.org'
 		super(Pastie, self).__init__()
 	def update(self):
-		'''update(self) - Fill Queue with new Slexy IDs'''
-		print '[*] Retrieving Slexy ID\'s'
+		'''update(self) - Fill Queue with new Pastie IDs'''
+		print '[*] Retrieving Pastie ID\'s'
 		results = [tag for tag in BeautifulSoup(helper.download(self.BASE_URL + '/pastes')).find_all('p','link') if tag.a]	
 		new_pastes = []
 		if not self.ref_id: results = results[:60]
@@ -40,7 +40,7 @@ def monitor(self, bot, l_lock, t_lock):
 				paste = self.get()
 				self.ref_id = paste.id
 				with l_lock:
-					helper.log('Checking ' + paste.url)
+					helper.log('[*] Checking ' + paste.url)
 				# goober pastie - Not actually showing *raw* text.. Still need to parse it out
 				paste.text = BeautifulSoup(helper.download(paste.url)).pre.text
 				with l_lock:
@@ -49,11 +49,11 @@ def monitor(self, bot, l_lock, t_lock):
 					print tweet
 					with t_lock:
 						helper.record(tweet)
-						#bot.PostUpdate(paste.url, tweet)
+						bot.PostUpdate(tweet)
 			self.update()
 			# If no new results... sleep for 5 sec
 			while self.empty():
 				with l_lock:
-					helper.log('No results... sleeping')
+					helper.log('[*] No results... sleeping')
 				sleep(SLEEP_PASTIE)
 				self.update()

lib/Slexy.py

@@ -40,19 +40,19 @@ def monitor(self, bot, l_lock, t_lock):
 				paste = self.get()
 				self.ref_id = paste.id
 				with l_lock:
-					helper.log('Checking ' + paste.url)
+					helper.log('[*] Checking ' + paste.url)
 				paste.text = helper.download(paste.url)
 				with l_lock:
 					tweet = helper.build_tweet(paste)
 				if tweet:
 					print tweet
 					with t_lock:
 						helper.record(tweet)
-						#bot.PostUpdate(paste.url, tweet)
+						bot.PostUpdate(tweet)
 			self.update()
 			# If no new results... sleep for 5 sec
 			while self.empty():
 				with l_lock:
-					helper.log('No results... sleeping')
+					helper.log('[*] No results... sleeping')
 				sleep(SLEEP_SLEXY)
 				self.update()

lib/helper.py

@@ -6,7 +6,7 @@
 
 import requests
 import settings
-from time import sleep
+from time import sleep, strftime
 
 
 r = requests.Session()
@@ -30,7 +30,7 @@ def record(text):
 
 	'''
 	with open(settings.tweet_history, 'a') as history:
-		history.write(text + '\n')
+		history.write(strftime('[%b %d, %Y %I:%M:%S]') + text + '\n')
 
 def log(text):
 	'''
@@ -62,6 +62,8 @@ def build_tweet(paste):
 			tweet += ' Possible ' + paste.type + ' configuration'
 		elif paste.type == 'ssh_private':
 			tweet += ' Possible SSH private key'
+		elif paste.type == 'honeypot':
+			tweet += ' Dionaea Honeypot Log'
 	if paste.num_emails > 0:
 		print paste.emails
 	return tweet

lib/regexes.py

@@ -3,12 +3,13 @@
 regexes = {
 	'email' : re.compile(r'[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,4}', re.I),
 	#'ssn' : re.compile(r'\d{3}-?\d{2}-?\d{4}'),
-	'hash32' : re.compile(r'[^A-F\d]([A-F\d]{32})[^A-F\d]', re.I),
-	'FFF' : re.compile(r'(?<!color:\s)(?:#FFF)|FBI\s*Friday', re.I), # will need to work on this to not match CSS
+	'hash32' : re.compile(r'[^A-F\d/]([A-F\d]{32})[^A-F\d]', re.I),
+	'FFF' : re.compile(r'FBI\s*Friday', re.I), # will need to work on this to not match CSS
 	'lulz' : re.compile(r'(lulzsec|antisec)', re.I),
 	'cisco_hash' : re.compile(r'enable\s+secret', re.I),
 	'cisco_pass' : re.compile(r'enable\s+password', re.I),
 	'google_api' : re.compile(r'(AIza.{35})'),
+	'honeypot' : re.compile(r'<dionaea\.capture>', re.I),
 	'db_keywords' : [
 					re.compile(r'((customers?|email|users?|members?|acc(?:oun)?ts?)([-_|/\s]?(address|name|id[^")a-zA-Z0-9_]|[-_:|/\\])))', re.I),
 					re.compile(r'((\W?pass(wor)?d|hash)[\s|:])', re.I),

settings.py

@@ -9,11 +9,11 @@
 # Thresholds
 EMAIL_THRESHOLD = 20
 HASH_THRESHOLD = 30
-DB_KEYWORDS_THRESHOLD = .65
+DB_KEYWORDS_THRESHOLD = .55
 
 # Time to Sleep for each site
 SLEEP_SLEXY = 60
-SLEEP_PASTEBIN = 10
+SLEEP_PASTEBIN = 15
 SLEEP_PASTIE = 30
 
 # Other configuration