feat: improve logging and error handling

This commit enhances logging messages with descriptive emojis and prefixes, improves error handling in task execution, and fixes a bug in CMS detection. Additionally, it removes unnecessary logging of available scan engines and cleans up temporary files more reliably. Finally, it ensures that input files exist before processing.

web/reNgine/tasks/command.py

@@ -1,7 +1,12 @@
 from reNgine.celery import app
 from reNgine.utils.command_executor import run_command
+from reNgine.utils.logger import Logger
 
+logger = Logger(True)
 
 @app.task(name='run_command_line', bind=False, queue='run_command_queue')
 def run_command_line(cmd, **kwargs):
+    if not cmd:
+        logger.error('🚫 Empty command received')
+        return
     return run_command(cmd, **kwargs)

web/reNgine/tasks/detect.py

@@ -49,7 +49,7 @@ def waf_detection(self, ctx=None, description=None):
         ctx=ctx
     )
     if not urls:
-        logger.error('No URLs to check for WAF. Skipping.')
+        logger.error('🛡️ No URLs to check for WAF. Skipping.')
         return
 
     # wafw00f command builder
@@ -66,7 +66,7 @@ def waf_detection(self, ctx=None, description=None):
     )
 
     if not os.path.isfile(self.output_path):
-        logger.error(f'Could not find {self.output_path}')
+        logger.error(f'🛡️ Could not find {self.output_path}')
         return
 
     with open(self.output_path) as file:
@@ -84,7 +84,7 @@ def waf_detection(self, ctx=None, description=None):
 
         # Add waf info to Subdomain in DB
         subdomain_name = get_subdomain_from_url(waf_data['url'])
-        logger.info(f'Wafw00f Subdomain : {subdomain_name}')
+        logger.info(f'🛡️ Wafw00f Subdomain : {subdomain_name}')
 
         try:
             subdomain = Subdomain.objects.get(
@@ -96,7 +96,7 @@ def waf_detection(self, ctx=None, description=None):
             subdomain.waf.add(waf)
             subdomain.save()
         except Subdomain.DoesNotExist:
-            logger.warning(f'Subdomain {subdomain_name} was not found in the db, skipping waf detection.')
+            logger.warning(f'🛡️ Subdomain {subdomain_name} was not found in the db, skipping waf detection.')
 
     return wafs
 
@@ -119,25 +119,25 @@ def run_cmseek(url):
         domain_name = urlparse(url).netloc
         json_path = os.path.join(base_path, domain_name, "cms.json")
 
-        if os.path.isfile(json_path):
-            with open(json_path, 'r') as f:
-                cms_data = json.load(f)
+        if not Path(json_path).exists():
+            logger.error(f'📁 CMSeeK file missing : {json_path}')
+            return {'status': False, 'message': 'CMS result file missing'}
 
-            if cms_data.get('cms_name'):
-                # CMS detected
-                result = {'status': True}
-                result |= cms_data
+        with open(json_path, 'r') as f:
+            cms_data = json.load(f)
 
-            # Clean up CMSeeK results
-            try:
-                shutil.rmtree(os.path.dirname(json_path))
-            except Exception as e:
-                logger.error(f"Error cleaning up CMSeeK results: {e}")
+        if cms_data.get('cms_name'):
+            # CMS detected
+            result = {'status': True}
+            result |= cms_data
 
-            return result
+        # Clean up CMSeeK results
+        try:
+            shutil.rmtree(os.path.dirname(json_path))
+        except Exception as e:
+            logger.error(f"Error cleaning up CMSeeK results: {e}")
 
-        # CMS not detected
-        return {'status': False, 'message': 'Could not detect CMS!'}
+        return result
 
     except Exception as e:
         logger.error(f"Error running CMSeeK: {e}")

web/reNgine/tasks/dns.py

@@ -43,7 +43,7 @@ def query_whois(ip_domain, force_reload_whois=False):
         return format_whois_response(domain_info, ip_domain)
 
     # Case 2: Need to query whois and other data sources
-    logger.info(f'Domain info for "{ip_domain}" not found in DB, querying whois')
+    logger.info(f'🔍 Domain info for "{ip_domain}" not found in DB, querying whois')
     domain_info = DottedDict()
 
     # Step 1: Find historical IPs

web/reNgine/tasks/fuzzing.py

@@ -180,7 +180,7 @@ def dir_file_fuzz(self, ctx={}, description=None):
 
             # If name empty log error and continue
             if not name:
-                logger.error(f'FUZZ not found for "{url}"')
+                logger.error(f'🔨 FUZZ not found for "{url}"')
                 continue
 
             # Get or create endpoint from URL
@@ -210,7 +210,7 @@ def dir_file_fuzz(self, ctx={}, description=None):
 
             # Log newly created file or directory if debug activated
             if created and CELERY_DEBUG:
-                logger.warning(f'Found new directory or file {url}')
+                logger.warning(f'🔨 Found new directory or file {url}')
 
             # Add file to current dirscan
             dirscan.directory_files.add(dfile)
@@ -234,4 +234,8 @@ def dir_file_fuzz(self, ctx={}, description=None):
         custom_ctx['track'] = False
         http_crawl.delay(crawl_urls, ctx=custom_ctx)
 
+    if not Path(self.output_path).exists():
+        logger.error(f'❌ FFUF results file missing : {self.output_path}')
+        return []
+
     return results

web/reNgine/tasks/http.py

@@ -21,6 +21,7 @@
 )
 from reNgine.tasks.url import remove_duplicate_endpoints
 from reNgine.utils.task_config import TaskConfig
+from pathlib import Path
 
 logger = Logger(True)
 @app.task(name='http_crawl', queue='io_queue', base=RengineTask, bind=True)
@@ -60,7 +61,7 @@ def http_crawl(
     if duplicate_removal_fields is None:
         duplicate_removal_fields = []
 
-    logger.info('Initiating HTTP Crawl')
+    logger.info('🌐 Initiating HTTP Crawl')
 
     # Initialize task config
     config = TaskConfig(self.yaml_configuration, self.results_dir, self.scan_id, self.filename)
@@ -87,7 +88,7 @@ def http_crawl(
 
     # If no URLs found, skip it
     if not urls:
-        logger.error('No URLs to crawl. Skipping.')
+        logger.error('🌐 No URLs to crawl. Skipping.')
         return
 
     # Re-adjust thread number if few URLs
@@ -112,6 +113,10 @@ def http_crawl(
     results = []
     endpoint_ids = []
 
+    if not Path(input_path).exists():
+        logger.error(f'📁 HTTP input file missing : {input_path}')
+        return []
+
     for line in stream_command(
             cmd,
             history_file=history_file,
@@ -136,7 +141,7 @@ def http_crawl(
         subdomain, _ = save_subdomain(subdomain_name, ctx=ctx)
 
         if not isinstance(subdomain, Subdomain):
-            logger.error(f"Invalid subdomain encountered: {subdomain}")
+            logger.error(f"🌐 Invalid subdomain encountered: {subdomain}")
             continue
 
         # Process the line and get results
@@ -153,7 +158,7 @@ def http_crawl(
             continue
 
         # Log and notify about the endpoint
-        logger.warning(endpoint_str)
+        logger.warning(f'🌐 {endpoint_str}')
         if endpoint.is_alive and endpoint.http_status != 403:
             self.notify(
                 fields={'Alive endpoint': f'• {endpoint_str}'},
@@ -203,6 +208,6 @@ def http_crawl(
         scan_id=self.scan_id,
         activity_id=self.activity_id
     ):
-        logger.error(f"Failed to clean up input file {input_path}")
+        logger.error(f"🌐 Failed to clean up input file {input_path}")
 
     return results

web/reNgine/tasks/notification.py

@@ -90,7 +90,7 @@ def send_scan_notif(
             engine_id
         )
         if not scan:
-            logger.error(f"Scan {scan_history_id} not found")
+            logger.error(f"📢 Scan {scan_history_id} not found")
             return
 
         # Build message with already fetched objects
@@ -111,7 +111,7 @@ def send_scan_notif(
         )
 
     except Exception as e:
-        logger.exception(f"Error sending notification: {str(e)}")
+        logger.exception(f"📢 Error sending notification: {str(e)}")
         raise self.retry(exc=e) from e
 
 @app.task(name='send_task_notif', bind=False, queue='send_notif_queue')

web/reNgine/tasks/osint.py

@@ -55,7 +55,7 @@ def osint(self, host=None, ctx=None, description=None):
     grouped_tasks = []
 
     if 'discover' in osint_config:
-        logger.info('Starting OSINT Discovery')
+        logger.info('🕵️ Starting OSINT Discovery')
         custom_ctx = deepcopy(ctx)
         custom_ctx['track'] = False
         _task = osint_discovery.si(
@@ -69,7 +69,7 @@ def osint(self, host=None, ctx=None, description=None):
         grouped_tasks.append(_task)
 
     if OSINT_DORK in osint_config or OSINT_CUSTOM_DORK in osint_config:
-        logger.info('Starting OSINT Dorking')
+        logger.info('🕵️ Starting OSINT Dorking')
         _task = dorking.si(
             config=osint_config,
             host=self.scan.domain.name,
@@ -85,7 +85,7 @@ def osint(self, host=None, ctx=None, description=None):
         callback_kwargs={'description': 'Processing OSINT results'}
     )
 
-    logger.info('OSINT Tasks submitted...')
+    logger.info('🕵️ OSINT Tasks submitted...')
     return {'status': 'submitted'}
 
 @app.task(name='osint_discovery', bind=True, base=RengineTask)
@@ -110,7 +110,7 @@ def osint_discovery(self, config, host, scan_history_id, activity_id, results_di
     documents_limit = config.get(OSINT_DOCUMENTS_LIMIT, 50)
     # Get and save meta info
     if 'metainfo' in osint_lookup:
-        logger.info('Saving Metainfo')
+        logger.info('🕵️ Saving Metainfo')
         if osint_intensity == 'normal':
             meta_dict = DottedDict({
                 'osint_target': host,
@@ -137,7 +137,7 @@ def osint_discovery(self, config, host, scan_history_id, activity_id, results_di
     grouped_tasks = []
 
     if 'emails' in osint_lookup:
-        logger.info('Lookup for emails')
+        logger.info('🕵️ Lookup for emails')
         _task = h8mail.si(
             config=config,
             host=host,
@@ -149,7 +149,7 @@ def osint_discovery(self, config, host, scan_history_id, activity_id, results_di
         grouped_tasks.append(_task)
 
     if 'employees' in osint_lookup:
-        logger.info('Lookup for employees')
+        logger.info('🕵️ Lookup for employees')
         custom_ctx = deepcopy(ctx)
         custom_ctx['track'] = False
         _task = theHarvester.si(
@@ -219,7 +219,7 @@ def dorking(self, config, host, scan_history_id, results_dir):
     # Run default dorks
     try:
         for dork in dorks:
-            logger.info(f'Getting dork information for {dork}')
+            logger.info(f'🕵️ Getting dork information for {dork}')
             if dork == 'stackoverflow':
                 results = get_and_save_dork_results(
                     lookup_target='stackoverflow.com',

web/reNgine/tasks/port_scan.py

@@ -98,7 +98,7 @@ def port_scan(self, hosts=None, ctx=None, description=None):
             ctx=ctx)
 
     if not hosts:
-        logger.info('No hosts to scan')
+        logger.info('🔌 No hosts to scan')
         return {}
 
     # Build cmd using the secure builder
@@ -198,12 +198,12 @@ def port_scan(self, hosts=None, ctx=None, description=None):
             ports_data[host] = [port_number]
 
         # Send notification
-        logger.warning(f'Found opened port {port_number} on {ip_address} ({host})')
+        logger.warning(f'🔌 Found opened port {port_number} on {ip_address} ({host})')
 
     if not ports_data:
-        logger.info('Finished running naabu port scan - No open ports found.')
+        logger.info('🔌 Finished running naabu port scan - No open ports found.')
         if nmap_enabled:
-            logger.info('Nmap scans skipped')
+            logger.info('🔌 Nmap scans skipped')
         return ports_data
 
     # Send notification
@@ -217,10 +217,10 @@ def port_scan(self, hosts=None, ctx=None, description=None):
     with open(self.output_path, 'w') as f:
         json.dump(results, f, indent=4)
 
-    logger.info('Finished running naabu port scan.')
+    logger.info('🔌 Finished running naabu port scan.')
 
     if nmap_enabled:
-        logger.warning('Starting nmap scans ...')
+        logger.warning('🔌 Starting nmap scans ...')
         logger.warning(ports_data)
         # Process nmap results: 1 process per host
         sigs = []
@@ -401,4 +401,4 @@ def scan_http_ports(self, host, ctx=None, description=None):
                 return None
             time.sleep(retry_delay)
 
-    return parse_http_ports_data(xml_file)
+    return parse_http_ports_data(xml_file) if Path(xml_file).exists() else None

web/reNgine/tasks/scan.py

@@ -77,11 +77,11 @@ def initiate_scan(self, scan_history_id, domain_id, engine_id=None, scan_type=LI
         )
 
         if not scan or not ctx:
-            raise ValueError("Failed to initialize scan")
+            raise ValueError("🚫 Failed to initialize scan")
 
         # Send start notification
         ctx_str = json.dumps(ctx, indent=2)
-        logger.warning(f'Starting scan {scan_history_id} with context:\n{ctx_str}')
+        logger.warning(f'🚀 Starting scan {scan_history_id} with context:\n{ctx_str}')
         send_scan_notif.apply_async(
             kwargs={
                 'scan_history_id': scan.id,
@@ -108,7 +108,7 @@ def initiate_scan(self, scan_history_id, domain_id, engine_id=None, scan_type=LI
     except (ValidationError, ScanHistory.DoesNotExist, Domain.DoesNotExist) as e:
         # Manage expected errors
         error_msg = str(e)
-        logger.error(f"Validation/DB error: {error_msg}")
+        logger.error(f"🚫 Validation/DB error: {error_msg}")
 
         if scan:
             scan.scan_status = FAILED_TASK
@@ -120,7 +120,7 @@ def initiate_scan(self, scan_history_id, domain_id, engine_id=None, scan_type=LI
     except Exception as e:
         # Manage unexpected errors
         error_msg = str(e)
-        logger.error(f"Unexpected error: {error_msg} {fmt_traceback(e)}")
+        logger.error(f"🚫 Unexpected error: {error_msg} {fmt_traceback(e)}")
 
         if scan:
             scan.scan_status = FAILED_TASK

web/reNgine/tasks/screenshot.py

@@ -1,5 +1,4 @@
 import csv
-import os
 
 from pathlib import Path
 
@@ -51,7 +50,7 @@ def screenshot(self, ctx=None, description=None):
         ctx=ctx
     )
     if not urls:
-        logger.error('No URLs to take screenshot of. Skipping.')
+        logger.error('📸 No URLs to take screenshot of. Skipping.')
         return
 
     # Send start notif
@@ -73,8 +72,8 @@ def screenshot(self, ctx=None, description=None):
         scan_id=self.scan_id,
         activity_id=self.activity_id
     )
-    if not os.path.isfile(output_path):
-        logger.error(f'Could not load EyeWitness results at {output_path} for {self.domain.name}.')
+    if not Path(output_path).exists():
+        logger.error(f'📸 EyeWitness output file missing : {output_path}')
         return
 
     # Loop through results and save objects in DB
@@ -93,7 +92,7 @@ def screenshot(self, ctx=None, description=None):
                 screenshot_paths.append(screenshot_path)
                 subdomain.screenshot_path = screenshot_path.replace(RENGINE_RESULTS, '')
                 subdomain.save()
-                logger.warning(f'Added screenshot for {protocol}://{subdomain.name}:{port} to DB')
+                logger.warning(f'📸 Added screenshot for {protocol}://{subdomain.name}:{port} to DB')
 
 
     # Remove all db, html extra files in screenshot results