Merge branch 'test'

Author: Mohamed Elbadry

Dockerfile

@@ -8,7 +8,8 @@ RUN pip3 install -r requirements.txt
 RUN go get -u github.com/melbadry9/subover
 RUN go get -u github.com/OJ/gobuster
 RUN go get -u github.com/tomnomnom/assetfinder
+RUN go get -u github.com/tomnomnom/httprobe
 ENV GOROOT=/root/go GOPATH=/go PATH=/root/go/bin:$PATH
-RUN wget wget https://github.com/OWASP/Amass/releases/download/v3.1.10/amass_v3.1.10_linux_amd64.zip;
+RUN wget https://github.com/OWASP/Amass/releases/download/v3.1.10/amass_v3.1.10_linux_amd64.zip
 RUN unzip -j amass_v3.1.10_linux_amd64.zip amass_v3.1.10_linux_amd64/amass -d ${GOROOT}/bin/
 CMD ["gunicorn", "-b", "0.0.0.0:8000", "app:Scan"]

README.md

@@ -1,6 +1,6 @@
 # ScanApi ![Python 3.5](https://img.shields.io/badge/Python-3.x-blue.svg) ![linux 64-bit](https://img.shields.io/badge/Linux-64bit-blue.svg)
 
-Subdomains-enumeration and subdomain-takeover monitoring api.
+Subdomains-enumeration, subdomain-takeover monitoring api and S3 bucket scanner.
 
 ## Installing
 
@@ -20,31 +20,50 @@ docker build -t scanapi:latest .
 docker run -d -p 8000:8000 scanapi
 ```
 
-- Add slack hook in `config.ini`
+- Update `config.ini` before building docker image.
 
-- For custom options edit `config.ini`
+- Add slack hook in `config.ini` if Slack is Enabled.
 
-## Running  
+- Commit docker image `docker commit <container id> scanapi:latest` to avoid losing data from db.
 
-- Open `http://127.0.0.1:8000/enum/domain/example.com/` in your browser.
-- Get Result `http://127.0.0.1:8000/db/domain/example.com/`.
-- S3 Scanner `http://127.0.0.1:8000/enum/s3/bucket-name/`
-- S3 Result `http://127.0.0.1:8000/db/s3/bucket-name/`
+## Endpoints  
+
+1. `/enum/domain/<domain>/`
+    - Start subdomain enumeration task in background then update db
+    - Domain ex: `example.com`
+
+2. `/enum/s3/<bucket-name>/`
+    - Start s3 bucket permissions scanner and update db
+    - Bucket-name ex: `example-prod`
+
+3. `/db/domain/<domain>/`
+    - Retrieve all subdomains from db if any exist
+
+4. `/db/domain/<domain>/?pro=http`
+    - Retrieve subdomains with port 80 opened from db if any exist
+
+5. `/db/domain/<domain>/?pro=https`
+    - Retrieve subdomains with port 443 opened from db if any exist
+
+6. `/db/s3/<bucket-name>/`
+    - Retrieve s3 bucket scanner data from db if any exist
 
 ## Supported Tools
 
 - [Amass](https://github.com/OWASP/Amass)
 - [Gasset](https://github.com/melbadry9/gasset)
 - [Subover](https://github.com/melbadry9/SubOver)
 - [Sublist3r](https://github.com/melbadry9/Sublist3r)
+- [Httprobe](https://github.com/tomnomnom/httprobe)
 - [Gobuster](https://github.com/OJ/gobuster)
 - [Assetfinder](https://github.com/tomnomnom/assetfinder)
 
 ## To-Do list
 
-- Add directory brute forcing monitoring.
-- Add open ports monitoring.
-- Add scheduling jobs.
+- [ ] Add directory brute forcing monitoring
+- [ ] Add open ports monitoring
+- [ ] Add scheduling jobs
+- [ ] Add UI
 
 ## Donation
 

app.py

@@ -1,6 +1,6 @@
 import json
 import multiprocessing
-from flask import Flask, jsonify, render_template, Response
+from flask import Flask, jsonify, render_template, Response, request
 
 import lib.core.log_handler
 from lib.core.config import config
@@ -30,7 +30,8 @@ def S3_check(bucket):
 
 @Scan.route("/db/domain/<domain>/")
 def Subdomain_from_db(domain):
-    return Response(get_subdomains(domain),200,mimetype="text/plain")
+    protocol = request.args.get("pro") or ""
+    return Response(get_subdomains(domain, protocol),200,mimetype="text/plain")
 
 @Scan.route("/db/s3/<bucket>/")
 def s3_from_db(bucket):

config.ini

@@ -5,21 +5,23 @@ debug = False
 secret = anything_12456547_1524556
 
 [SLACK]
-enabled = False
 hook = 
+enabled = False
+
 
 [COOKIE]
 # gasset cookie to enable censys and virustotal
 fb_cookie = 
 
 [GENERAL]
-threads = 20
+threads = 30
 resolver = 8.8.8.8
 
 [TOOLS]
 amass = True
 gasset = True
 subover = True
+httprobe = True
 gobuster = True
 sublist3r = True
 assetfinder = True
@@ -45,5 +47,6 @@ s3transfer = False
 file_path = app.log
 
 [aws]
+# aws keys 
 id = 
 secret = 

install.sh

@@ -7,6 +7,7 @@ export PATH=$PATH:$GOROOT/bin:$GOPATH/bin;
 go get -u github.com/melbadry9/subover;
 go get -u github.com/OJ/gobuster;
 go get -u github.com/tomnomnom/assetfinder;
+go get -u github.com/tomnomnom/httprobe;
 wget https://github.com/OWASP/Amass/releases/download/v3.1.10/amass_v3.1.10_linux_amd64.zip;
 unzip amass_v3.1.10_linux_amd64.zip;
 cp amass_v3.1.10_linux_amd64/amass $GOPATH/bin/;

lib/core/database.py

@@ -75,7 +75,7 @@ def __del__(self):
 
     def create_db(self):
         with self.lock:
-            self.cdb.executescript("""CREATE TABLE IF NOT EXISTS "domains" ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, `main_domain` TEXT NOT NULL , `sub_domain` TEXT )""")
+            self.cdb.executescript("""CREATE TABLE IF NOT EXISTS "domains" ( `id` INTEGER NOT NULL PRIMARY KEY AUTOINCREMENT UNIQUE, `main_domain` TEXT NOT NULL, `sub_domain` TEXT, `http` INTEGER NOT NULL DEFAULT 0, `https` INTEGER NOT NULL DEFAULT 0 )""")
             self.db.commit()
 
     def insert_domains(self, sub_domain:list):
@@ -89,5 +89,21 @@ def read_domains(self):
         self.cdb.execute("select sub_domain from domains where main_domain = ?",(self.domain,))
         return [i[0] for i in self.cdb.fetchall()]
 
+    def update_protocol(self, protocol:str, sub_domain:list):
+        with self.lock:
+            for item in sub_domain:
+                if protocol == "http":
+                    self.cdb.execute("update domains set  http = 1 where sub_domain = ?",(item,))
+                elif protocol == "https":
+                    self.cdb.execute("update domains set  https = 1 where sub_domain = ?",(item,))
+            self.db.commit()
+
+    def read_domains_protocol(self, protocol:str):
+        if protocol == "http":
+            self.cdb.execute("select sub_domain from domains where main_domain = ? and http=1",(self.domain,))
+        elif protocol == "https":
+            self.cdb.execute("select sub_domain from domains where main_domain = ? and https=1",(self.domain,))
+        return [i[0] for i in self.cdb.fetchall()]
+    
     def new_domains(self):
         return list(set(self.read_domains()) - set(self.old_sub))

lib/core/opp.py

@@ -64,4 +64,11 @@ def __init__(self, domain):
         ProcessBase.__init__(self)
         self.name = "GoBusterDNS"
         self.command = "gobuster dns -z -q -d {0} -r {1} -w {2} -t {3}".format(domain, config['GENERAL']['resolver'], DNS_LIST, self.threads)
-        self.pattern = r"Found: (.+)\n"
+        self.pattern = r"Found: (.+)\n"
+
+class Httprobe(ProcessBase):
+    def __init__(self, file):
+        ProcessBase.__init__(self)
+        self.name = "Httprobe"
+        self.command = "cat {0} | httprobe -c {1}".format(file, self.threads)
+        self.pattern = r"(.+)\n"

lib/scan/subdomain_job.py

@@ -8,8 +8,8 @@
 from ..core.database import SubDomainData
 from ..core.log_handler import scan_logger
 from ..thirdparty.Gasset.asset import main as Gasset
-from ..core.opp import SubOver, GoBuster, AssetFinder, Amass, GoBusterDNS
 from ..thirdparty.Sublist3r.sublist3r import main as Sublist3r
+from ..core.opp import SubOver, GoBuster, AssetFinder, Amass, GoBusterDNS, Httprobe
 
 
 def sub_job(domain):
@@ -78,15 +78,15 @@ def sub_job(domain):
     # All Subdomains
     final_list = clean(set(final_list))
     meta_data['count'] = len(final_list)
-    
-    # SubOver
-    if config['TOOLS'].getboolean('subover'):
 
-        # Temp file
-        temp_file = tempfile.NamedTemporaryFile("w+t", encoding="utf-8")
-        for dom in final_list:
-            temp_file.writelines(dom + "\n")
+     # Temp file
+    temp_file = tempfile.NamedTemporaryFile("w+t", encoding="utf-8", delete=False)
+    for item in final_list:
+        temp_file.writelines(item + "\n")
+    temp_file.seek(0)
 
+    # SubOver
+    if config['TOOLS'].getboolean('subover'):
         try:
             pro_subover = SubOver(temp_file.name)
             data = pro_subover.exec_command()
@@ -98,15 +98,31 @@ def sub_job(domain):
             error_msg = "SubOver: " + str(e)
             scan_logger.error(error_msg, exc_info=True)
             final_error.append(error_msg)
-
+    
     DB.insert_domains(final_list)
     new_subs = DB.new_domains()
 
-    # Bypass first run 
+    # Httprobe
+    if config['TOOLS'].getboolean('httprobe'):
+        try:
+            http_probe = Httprobe(temp_file.name)
+            data = http_probe.exec_command()
+            alive_data = data['Httprobe']['data']
+            https_domain = [dom.replace("https://","") for dom in alive_data if dom.startswith("https://")]
+            http_domain = [dom.replace("http://","") for dom in alive_data if dom.startswith("http://")]
+            DB.update_protocol("http", http_domain)
+            DB.update_protocol("https", https_domain)
+        except Exception as e:
+            error_msg = "Httprobe: " + str(e)
+            scan_logger.error(error_msg, exc_info=True)
+            final_error.append(error_msg)
+
+    # Bypass first run and  avoid huge host on slack webhook
     if not len(new_subs) == len(final_list):
         if not len(new_subs) == 0:
             meta_data['new_count'] = len(new_subs)
-            meta_data['subdomains'] = new_subs
+            if not len(new_subs) > 200:
+                meta_data['subdomains'] = new_subs
 
     # Add Errors
     if len(set(final_error) - {""}) > 0:
@@ -120,7 +136,12 @@ def sub_job(domain):
     scan_logger.info(json.dumps(meta_data, indent=4))
     return meta_data
 
-def get_subdomains(domain):
+def get_subdomains(domain, protocol):
     DB = SubDomainData(domain)
-    subs = DB.read_domains()
+    if protocol == "http":
+        subs = DB.read_domains_protocol("http")
+    elif protocol == "https":
+        subs = DB.read_domains_protocol("https")
+    else:
+        subs = DB.read_domains()
     return '\n'.join(subs)