Adding findomain & subfinder

Author: Mohamed Elbadry

README.md

@@ -52,6 +52,8 @@ docker run -d -p 8000:8000 scanapi
 
 - [Amass](https://github.com/OWASP/Amass)
 - [Gasset](https://github.com/melbadry9/gasset)
+- [Findomain](https://github.com/Edu4rdSHL/findomain)
+- [Subfinder](https://github.com/projectdiscovery/subfinder)
 - [Subover](https://github.com/melbadry9/SubOver)
 - [Sublist3r](https://github.com/melbadry9/Sublist3r)
 - [Httprobe](https://github.com/tomnomnom/httprobe)

app.py

@@ -1,8 +1,6 @@
-import json
 import multiprocessing
 from flask import Flask, jsonify, render_template, Response, request
 
-import lib.core.log_handler
 from lib.core.config import config
 from lib.scan.s3_job import s3_job, get_s3
 from lib.scan.subdomain_job import sub_job, get_subdomains

config.ini

@@ -11,7 +11,8 @@ enabled = False
 
 [COOKIE]
 # gasset cookie to enable censys and virustotal
-fb_cookie = 
+shodan = ''
+fb_cookie = '' 
 
 [GENERAL]
 threads = 30
@@ -23,6 +24,8 @@ gasset = True
 subover = True
 httprobe = True
 gobuster = True
+subfinder = True
+findomain = True
 sublist3r = True
 assetfinder = True
 gobusterDNS = False
@@ -33,7 +36,8 @@ dns = small
 dir = small
 
 [LOGGING]
-# disable modules logging 
+# disable modules logging
+db = True
 s3 = True
 opp = True
 slack = True

install.sh

@@ -8,6 +8,10 @@ go get -u github.com/melbadry9/subover;
 go get -u github.com/OJ/gobuster;
 go get -u github.com/tomnomnom/assetfinder;
 go get -u github.com/tomnomnom/httprobe;
+go get -u -v github.com/projectdiscovery/subfinder/cmd/subfinder;
 wget https://github.com/OWASP/Amass/releases/download/v3.1.10/amass_v3.1.10_linux_amd64.zip;
+wget https://github.com/Edu4rdSHL/findomain/releases/latest/download/findomain-linux;
+chmod +x findomain-linux;
 unzip amass_v3.1.10_linux_amd64.zip;
+cp findomain-linux $GOPATH/bin/findomain;
 cp amass_v3.1.10_linux_amd64/amass $GOPATH/bin/;

lib/core/database.py

@@ -1,12 +1,14 @@
-import os
 import json
 import sqlite3
+import logging
 
-from pathlib2 import Path
 from threading import Lock
 from ..paths.helper import DB_FILE
 
 
+database = logging.getLogger("db")
+database.addHandler(logging.NullHandler())
+
 class DataBase(object):
     lock = Lock()
     def __init__(self):
@@ -72,31 +74,40 @@ def __init__(self, domain):
 
     def __del__(self):
         self.db.close()
+
+    def Save(self):
+        self.db.commit()
 
     def create_db(self):
         with self.lock:
             self.cdb.executescript("""CREATE TABLE IF NOT EXISTS "domains" ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, `main_domain` TEXT NOT NULL, `sub_domain` TEXT, `http` INTEGER NOT NULL DEFAULT 0, `https` INTEGER NOT NULL DEFAULT 0 )""")
-            self.db.commit()
+            self.Save()
 
     def insert_domains(self, sub_domain:list):
         with self.lock:
             for item in sub_domain:
                 if item not in self.old_sub:
                     self.cdb.execute("insert into domains (main_domain,sub_domain) values (?,?)",(self.domain,item))
-            self.db.commit()
+            self.Save()
 
     def read_domains(self):
         self.cdb.execute("select sub_domain from domains where main_domain = ?",(self.domain,))
         return [i[0] for i in self.cdb.fetchall()]
 
     def update_protocol(self, protocol:str, sub_domain:list):
         with self.lock:
-            for item in sub_domain:
-                if protocol == "http":
-                    self.cdb.execute("update domains set  http = 1 where sub_domain = ?",(item,))
-                elif protocol == "https":
-                    self.cdb.execute("update domains set  https = 1 where sub_domain = ?",(item,))
-            self.db.commit()
+            done = False
+            while not done:
+                try:
+                    for item in sub_domain:
+                        if protocol == "http":
+                            self.cdb.execute("update domains set  http = 1 where sub_domain = ?",(item,))
+                        elif protocol == "https":
+                            self.cdb.execute("update domains set  https = 1 where sub_domain = ?",(item,))
+                    done = True
+                except Exception as e:
+                    database.error("Error while updateing db\n {0}".format(e), stack_info=True)
+
 
     def read_domains_protocol(self, protocol:str):
         if protocol == "http":

lib/core/log_handler.py

@@ -6,6 +6,7 @@
 logging.basicConfig(filename=config['LOGGING']['file_path'], filemode='w', format='%(asctime)s [%(levelname)s] [%(name)s] %(message)s', datefmt='%d-%b-%y %H:%M:%S', level=logging.DEBUG)
 
 loggers = [
+    'db',
     's3', 
     'opp', 
     'slack', 

lib/core/opp.py

@@ -4,8 +4,6 @@
 
 from ..core.config import config
 from ..paths.setter import DIR_LIST, DNS_LIST
-from ..thirdparty.Gasset.asset import main as Gasset
-from ..thirdparty.Sublist3r.sublist3r import main as Sublist3r
 
 opp_logger = logging.getLogger("opp")
 opp_logger.addHandler(logging.NullHandler())
@@ -59,6 +57,20 @@ def __init__(self, domain):
         self.command = "amass enum -passive -d {0}".format(domain)
         self.pattern = r"(.+)\n"
 
+class Findomain(ProcessBase):
+    def __init__(self, domain):
+        ProcessBase.__init__(self)
+        self.name = "Findomain"
+        self.command = "findomain -t {0} -q".format(domain)
+        self.pattern = r"(.+)\n"
+
+class Subfinder(ProcessBase):
+    def __init__(self, domain):
+        ProcessBase.__init__(self)
+        self.name = "Subfinder"
+        self.command = "subfinder -d {0} -silent".format(domain)
+        self.pattern = r"(.+)\n"
+
 class GoBusterDNS(ProcessBase):
     def __init__(self, domain):
         ProcessBase.__init__(self)

lib/core/slack.py

@@ -1,5 +1,4 @@
 import time
-import json
 import logging
 import requests
 

lib/scan/subdomain_job.py

@@ -9,7 +9,7 @@
 from ..core.log_handler import scan_logger
 from ..thirdparty.Gasset.asset import main as Gasset
 from ..thirdparty.Sublist3r.sublist3r import main as Sublist3r
-from ..core.opp import SubOver, GoBuster, AssetFinder, Amass, GoBusterDNS, Httprobe
+from ..core.opp import SubOver, GoBuster, AssetFinder, Amass, GoBusterDNS, Httprobe, Findomain, Subfinder
 
 
 def sub_job(domain):
@@ -52,6 +52,30 @@ def sub_job(domain):
             error_msg = "AssetFinder: " + str(e)
             scan_logger.error(error_msg, exc_info=True)
             final_error.append(error_msg)
+
+    # Findomain
+    if config['TOOLS'].getboolean('findomain'):
+        try:
+            pro_findomain_finder = Findomain(domain)
+            data = pro_findomain_finder.exec_command()
+            final_list.extend(data['Findomain']['data'])
+            final_error.append(data['Findomain']['error'])
+        except Exception as e:
+            error_msg = "Findomain: " + str(e)
+            scan_logger.error(error_msg, exc_info=True)
+            final_error.append(error_msg)
+    
+    # Subfinder
+    if config['TOOLS'].getboolean('subfinder'):
+        try:
+            pro_subfinder_finder = Subfinder(domain)
+            data = pro_subfinder_finder.exec_command()
+            final_list.extend(data['Subfinder']['data'])
+            final_error.append(data['Subfinder']['error'])
+        except Exception as e:
+            error_msg = "Subfinder: " + str(e)
+            scan_logger.error(error_msg, exc_info=True)
+            final_error.append(error_msg)
 
     # Amass    
     if config['TOOLS'].getboolean('amass'):
@@ -112,6 +136,7 @@ def sub_job(domain):
             http_domain = [dom.replace("http://","") for dom in alive_data if dom.startswith("http://")]
             DB.update_protocol("http", http_domain)
             DB.update_protocol("https", https_domain)
+            DB.Save()
         except Exception as e:
             error_msg = "Httprobe: " + str(e)
             scan_logger.error(error_msg, exc_info=True)

lib/thirdparty/Gasset/asset.py

@@ -145,6 +145,7 @@ def __init__(self, domain, shared=None):
         self.url = self.BASE_URL.format(domain=domain)
 
     def SendRequest(self, url):
+        self.HEADERS['Cookie'] = config["COOKIE"]["shodan"]
         return self.session.get(url, stream=True)
 
     def HandleResponse(self, res):