dashboard.py

#!/usr/bin/env python3
"""
Dark Web Threat Intelligence Dashboard

A Streamlit-based dashboard for visualizing and managing 
dark web scraping results.

Author: AI Assistant
License: MIT
"""

import streamlit as st
import pandas as pd
import sqlite3
import json
import matplotlib.pyplot as plt
import seaborn as sns
from datetime import datetime, timedelta
import plotly.express as px
import plotly.graph_objects as go
from pathlib import Path
import subprocess
import sys
import os

# Page configuration
st.set_page_config(
    page_title="Dark Web Threat Intelligence",
    page_icon="🕵️",
    layout="wide",
    initial_sidebar_state="expanded"
)

class DashboardManager:
    """Manages the dashboard functionality."""
    
    def __init__(self):
        self.config_file = "config.json"
        self.csv_file = "results.csv"
        self.db_file = "results.db"
        
    def load_config(self):
        """Load configuration from JSON file."""
        try:
            with open(self.config_file, 'r') as f:
                return json.load(f)
        except FileNotFoundError:
            st.error(f"Configuration file {self.config_file} not found!")
            return None
    
    def load_results_csv(self):
        """Load results from CSV file."""
        try:
            if os.path.exists(self.csv_file):
                df = pd.read_csv(self.csv_file)
                df['timestamp'] = pd.to_datetime(df['timestamp'])
                return df
            else:
                return pd.DataFrame(columns=['timestamp', 'url', 'keyword', 'snippet', 'match_position'])
        except Exception as e:
            st.error(f"Error loading CSV: {e}")
            return pd.DataFrame()
    
    def load_results_sqlite(self):
        """Load results from SQLite database."""
        try:
            if os.path.exists(self.db_file):
                conn = sqlite3.connect(self.db_file)
                df = pd.read_sql_query("SELECT * FROM results ORDER BY timestamp DESC", conn)
                df['timestamp'] = pd.to_datetime(df['timestamp'])
                conn.close()
                return df
            else:
                return pd.DataFrame(columns=['timestamp', 'url', 'keyword', 'snippet', 'match_position'])
        except Exception as e:
            st.error(f"Error loading database: {e}")
            return pd.DataFrame()
    
    def run_manual_scan(self):
        """Run a manual scan."""
        try:
            with st.spinner("Running scan... This may take several minutes."):
                result = subprocess.run([sys.executable, "main.py"], 
                                      capture_output=True, text=True, timeout=300)
                if result.returncode == 0:
                    st.success("Scan completed successfully!")
                    st.text("Output:")
                    st.code(result.stdout)
                else:
                    st.error("Scan failed!")
                    st.code(result.stderr)
        except subprocess.TimeoutExpired:
            st.error("Scan timed out after 5 minutes.")
        except Exception as e:
            st.error(f"Error running scan: {e}")

def main():
    """Main dashboard function."""
    
    # Initialize dashboard manager
    dashboard = DashboardManager()
    
    # Header
    st.title("🕵️ Dark Web Threat Intelligence Dashboard")
    st.markdown("---")
    
    # Sidebar
    st.sidebar.title("Navigation")
    page = st.sidebar.selectbox("Select Page", [
        "Overview", 
        "Results Analysis", 
        "Configuration", 
        "Manual Scan",
        "Real-time Monitoring"
    ])
    
    # Load configuration
    config = dashboard.load_config()
    if not config:
        st.error("Cannot load configuration. Make sure config.json exists.")
        return
    
    # Load results
    if config['settings']['output_format'] == 'sqlite':
        df = dashboard.load_results_sqlite()
    else:
        df = dashboard.load_results_csv()
    
    if page == "Overview":
        show_overview(df, config)
    elif page == "Results Analysis":
        show_results_analysis(df)
    elif page == "Configuration":
        show_configuration(config)
    elif page == "Manual Scan":
        show_manual_scan(dashboard)
    elif page == "Real-time Monitoring":
        show_realtime_monitoring(df)

def show_overview(df, config):
    """Show overview page."""
    st.header("📊 Overview")
    
    # Metrics
    col1, col2, col3, col4 = st.columns(4)
    
    with col1:
        st.metric("Total Matches", len(df))
    
    with col2:
        unique_keywords = df['keyword'].nunique() if not df.empty else 0
        st.metric("Unique Keywords", unique_keywords)
    
    with col3:
        unique_sites = df['url'].nunique() if not df.empty else 0
        st.metric("Sites Scanned", unique_sites)
    
    with col4:
        if not df.empty:
            last_scan = df['timestamp'].max().strftime("%Y-%m-%d %H:%M") if pd.notna(df['timestamp'].max()) else "Never"
        else:
            last_scan = "Never"
        st.metric("Last Scan", last_scan)
    
    # Recent activity
    st.subheader("🕐 Recent Activity")
    if not df.empty:
        recent_df = df.head(10)
        st.dataframe(recent_df[['timestamp', 'keyword', 'url', 'snippet']], use_container_width=True)
    else:
        st.info("No data available. Run a scan to see results.")
    
    # Keyword frequency chart
    if not df.empty:
        st.subheader("📈 Keyword Frequency")
        keyword_counts = df['keyword'].value_counts().head(10)
        
        fig = px.bar(
            x=keyword_counts.values, 
            y=keyword_counts.index,
            orientation='h',
            labels={'x': 'Count', 'y': 'Keyword'},
            title="Top 10 Keywords Found"
        )
        fig.update_layout(height=400)
        st.plotly_chart(fig, use_container_width=True)

def show_results_analysis(df):
    """Show detailed results analysis."""
    st.header("🔍 Results Analysis")
    
    if df.empty:
        st.info("No data available for analysis.")
        return
    
    # Filters
    st.subheader("🔧 Filters")
    col1, col2, col3 = st.columns(3)
    
    with col1:
        keywords = st.multiselect("Keywords", df['keyword'].unique())
    
    with col2:
        date_range = st.date_input(
            "Date Range",
            value=(df['timestamp'].min().date(), df['timestamp'].max().date()),
            min_value=df['timestamp'].min().date(),
            max_value=df['timestamp'].max().date()
        )
    
    with col3:
        sites = st.multiselect("Sites", df['url'].unique())
    
    # Apply filters
    filtered_df = df.copy()
    
    if keywords:
        filtered_df = filtered_df[filtered_df['keyword'].isin(keywords)]
    
    if len(date_range) == 2:
        start_date, end_date = date_range
        filtered_df = filtered_df[
            (filtered_df['timestamp'].dt.date >= start_date) & 
            (filtered_df['timestamp'].dt.date <= end_date)
        ]
    
    if sites:
        filtered_df = filtered_df[filtered_df['url'].isin(sites)]
    
    # Display filtered results
    st.subheader(f"📋 Filtered Results ({len(filtered_df)} matches)")
    st.dataframe(filtered_df, use_container_width=True)
    
    # Time series analysis
    if not filtered_df.empty:
        st.subheader("📅 Activity Timeline")
        daily_counts = filtered_df.groupby(filtered_df['timestamp'].dt.date).size()
        
        fig = px.line(
            x=daily_counts.index, 
            y=daily_counts.values,
            labels={'x': 'Date', 'y': 'Matches Found'},
            title="Daily Activity"
        )
        st.plotly_chart(fig, use_container_width=True)
        
        # Heatmap
        st.subheader("🔥 Keyword-Site Heatmap")
        if len(filtered_df) > 0:
            heatmap_data = filtered_df.pivot_table(
                index='keyword', 
                columns='url', 
                values='match_position', 
                aggfunc='count', 
                fill_value=0
            )
            
            fig, ax = plt.subplots(figsize=(12, 8))
            sns.heatmap(heatmap_data, annot=True, cmap='YlOrRd', ax=ax)
            plt.title("Keyword Matches by Site")
            plt.xticks(rotation=45)
            plt.yticks(rotation=0)
            st.pyplot(fig)

def show_configuration(config):
    """Show configuration page."""
    st.header("⚙️ Configuration")
    
    # Display current configuration
    st.subheader("📋 Current Settings")
    
    # Tor settings
    st.write("**Tor Proxy:**")
    st.write(f"- Host: {config['settings']['tor_proxy']['host']}")
    st.write(f"- Port: {config['settings']['tor_proxy']['port']}")
    
    # Keywords
    st.write("**Keywords:**")
    for i, keyword in enumerate(config['keywords'], 1):
        st.write(f"{i}. {keyword}")
    
    # Target sites
    st.write("**Target Sites:**")
    for i, site in enumerate(config['target_sites'], 1):
        st.write(f"{i}. {site}")
    
    # Alerts
    st.write("**Alerts:**")
    st.write(f"- Email: {'Enabled' if config['alerts']['email']['enabled'] else 'Disabled'}")
    st.write(f"- Telegram: {'Enabled' if config['alerts']['telegram']['enabled'] else 'Disabled'}")
    
    # Configuration editor
    st.subheader("✏️ Edit Configuration")
    
    with st.expander("Edit Keywords"):
        new_keywords = st.text_area(
            "Keywords (one per line)", 
            value='\n'.join(config['keywords'])
        )
        
        if st.button("Update Keywords"):
            config['keywords'] = [k.strip() for k in new_keywords.split('\n') if k.strip()]
            try:
                with open('config.json', 'w') as f:
                    json.dump(config, f, indent=2)
                st.success("Keywords updated successfully!")
                st.experimental_rerun()
            except Exception as e:
                st.error(f"Error updating keywords: {e}")
    
    with st.expander("Edit Target Sites"):
        new_sites = st.text_area(
            "Target Sites (one per line)", 
            value='\n'.join(config['target_sites'])
        )
        
        if st.button("Update Sites"):
            config['target_sites'] = [s.strip() for s in new_sites.split('\n') if s.strip()]
            try:
                with open('config.json', 'w') as f:
                    json.dump(config, f, indent=2)
                st.success("Target sites updated successfully!")
                st.experimental_rerun()
            except Exception as e:
                st.error(f"Error updating sites: {e}")

def show_manual_scan(dashboard):
    """Show manual scan page."""
    st.header("🔄 Manual Scan")
    
    st.write("Run a manual scan of all configured sites for threat intelligence.")
    
    # Scan status
    if os.path.exists("scraper.log"):
        with open("scraper.log", "r") as f:
            log_content = f.read()
        
        st.subheader("📋 Recent Log Entries")
        st.text_area("Log Output", log_content.split('\n')[-20:], height=200)
    
    # Manual scan button
    if st.button("🚀 Start Manual Scan", type="primary"):
        dashboard.run_manual_scan()
        st.experimental_rerun()
    
    # Test Tor connection
    if st.button("🧪 Test Tor Connection"):
        try:
            result = subprocess.run([sys.executable, "main.py", "--test-tor"], 
                                  capture_output=True, text=True, timeout=30)
            if "successful" in result.stdout:
                st.success("✅ Tor connection successful!")
            else:
                st.error("❌ Tor connection failed!")
            st.code(result.stdout)
        except Exception as e:
            st.error(f"Error testing Tor: {e}")

def show_realtime_monitoring(df):
    """Show real-time monitoring page."""
    st.header("⚡ Real-time Monitoring")
    
    # Auto-refresh
    if st.checkbox("Auto-refresh (30 seconds)"):
        import time
        placeholder = st.empty()
        
        while True:
            with placeholder.container():
                if not df.empty:
                    # Recent activity
                    st.subheader("🕐 Latest Activity")
                    recent = df.head(5)
                    st.dataframe(recent[['timestamp', 'keyword', 'url']])
                    
                    # Live metrics
                    col1, col2, col3 = st.columns(3)
                    
                    with col1:
                        today_count = len(df[df['timestamp'].dt.date == datetime.now().date()])
                        st.metric("Today's Matches", today_count)
                    
                    with col2:
                        last_hour_count = len(df[df['timestamp'] > datetime.now() - timedelta(hours=1)])
                        st.metric("Last Hour", last_hour_count)
                    
                    with col3:
                        if not df.empty:
                            last_activity = (datetime.now() - df['timestamp'].max()).total_seconds() / 60
                            st.metric("Minutes Since Last Match", f"{last_activity:.0f}")
                
                else:
                    st.info("No data available for monitoring.")
            
            time.sleep(30)
            st.experimental_rerun()
    
    else:
        st.info("Enable auto-refresh to monitor activity in real-time.")

if __name__ == "__main__":
    main()