docker-compose.yml

version: '3.8'

x-logging: &default-logging
  driver: json-file
  options:
    max-size: "10m"
    max-file: "3"

services:
  # PostgreSQL Database
  postgres:
    image: postgres:15-alpine
    container_name: mlops-postgres
    environment:
      POSTGRES_DB: ${POSTGRES_DB:-mlops}
      POSTGRES_USER: ${POSTGRES_USER:-mlops_user}
      POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-mlops_password}
    ports:
      - "5432:5432"
    volumes:
      - postgres_data:/var/lib/postgresql/data
      - ./init.sql:/docker-entrypoint-initdb.d/init.sql
    networks:
      - mlops-network
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-mlops_user}"]
      interval: 10s
      timeout: 5s
      retries: 5
    logging: *default-logging

  # Redis Cache
  redis:
    image: redis:7-alpine
    container_name: mlops-redis
    ports:
      - "6379:6379"
    volumes:
      - redis_data:/data
    command: redis-server --appendonly yes --requirepass ${REDIS_PASSWORD:-}
    networks:
      - mlops-network
    healthcheck:
      test: ["CMD", "redis-cli", "--raw", "incr", "ping"]
      interval: 10s
      timeout: 3s
      retries: 3
    logging: *default-logging

  # MLflow Tracking Server
  mlflow-tracking:
    build:
      context: .
      dockerfile: infrastructure/Dockerfile.mlflow.enhanced
    container_name: mlops-mlflow
    environment:
      MLFLOW_TRACKING_URI: http://mlflow-tracking:5000
      MLFLOW_BACKEND_STORE_URI: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
      MLFLOW_ARTIFACT_ROOT: s3://${AWS_S3_BUCKET:-mlops-artifacts}/mlflow
      MLFLOW_S3_ENDPOINT_URL: ${AWS_S3_ENDPOINT_URL:-}
      AWS_ACCESS_KEY_ID: ${AWS_ACCESS_KEY_ID:-}
      AWS_SECRET_ACCESS_KEY: ${AWS_SECRET_ACCESS_KEY:-}
      AWS_REGION: ${AWS_REGION:-us-east-1}
    ports:
      - "5000:5000"
    volumes:
      - ./mlruns:/mlruns
      - ./artifacts:/artifacts
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - mlops-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:5000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    logging: *default-logging

  # Prefect Server
  prefect-server:
    image: prefecthq/prefect:2-python3.9
    container_name: mlops-prefect
    environment:
      PREFECT_API_URL: http://prefect-server:4200/api
      PREFECT_SERVER_API_HOST: 0.0.0.0
      PREFECT_LOGGING_LEVEL: INFO
      PREFECT_LOGGING_FORMAT: json
      PREFECT_ORION_DATABASE_CONNECTION_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
    ports:
      - "4200:4200"
    command: prefect server start
    depends_on:
      postgres:
        condition: service_healthy
    networks:
      - mlops-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:4200/api/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    logging: *default-logging

  # FastAPI Application
  ml-api:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: mlops-api
    environment:
      MLFLOW_TRACKING_URI: http://mlflow-tracking:5000
      PREFECT_API_URL: http://prefect-server:4200/api
      API_HOST: 0.0.0.0
      API_PORT: 8000
      DATABASE_URL: postgresql://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB}
      REDIS_URL: redis://:${REDIS_PASSWORD}@redis:6379/0
      LOG_LEVEL: ${LOG_LEVEL:-INFO}
      DEBUG: ${DEBUG:-false}
    ports:
      - "8000:8000"
    volumes:
      - ./src:/app/src
      - ./logs:/app/logs
    depends_on:
      mlflow-tracking:
        condition: service_healthy
      prefect-server:
        condition: service_healthy
      redis:
        condition: service_healthy
    networks:
      - mlops-network
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8000/health"]
      interval: 30s
      timeout: 10s
      retries: 3
    logging: *default-logging

  # Monitoring Dashboard
  monitoring:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: mlops-monitoring
    environment:
      MLFLOW_TRACKING_URI: http://mlflow-tracking:5000
      API_URL: http://ml-api:8000
      STREAMLIT_SERVER_PORT: 8501
      STREAMLIT_SERVER_ADDRESS: 0.0.0.0
    ports:
      - "8501:8501"
    volumes:
      - ./src:/app/src
    command: streamlit run src/monitoring/dashboard.py --server.port 8501 --server.address 0.0.0.0
    depends_on:
      ml-api:
        condition: service_healthy
    networks:
      - mlops-network
    logging: *default-logging

  # Prometheus for metrics
  prometheus:
    image: prom/prometheus:latest
    container_name: mlops-prometheus
    ports:
      - "9090:9090"
    volumes:
      - ./infrastructure/prometheus.yml:/etc/prometheus/prometheus.yml
      - prometheus_data:/prometheus
    command:
      - '--config.file=/etc/prometheus/prometheus.yml'
      - '--storage.tsdb.path=/prometheus'
      - '--web.console.libraries=/etc/prometheus/console_libraries'
      - '--web.console.templates=/etc/prometheus/consoles'
      - '--storage.tsdb.retention.time=200h'
      - '--web.enable-lifecycle'
    networks:
      - mlops-network
    logging: *default-logging

  # Grafana for visualization
  grafana:
    image: grafana/grafana:latest
    container_name: mlops-grafana
    ports:
      - "3000:3000"
    environment:
      GF_SECURITY_ADMIN_PASSWORD: ${GRAFANA_PASSWORD:-admin}
      GF_INSTALL_PLUGINS: grafana-piechart-panel
    volumes:
      - grafana_data:/var/lib/grafana
      - ./infrastructure/grafana/dashboards:/etc/grafana/provisioning/dashboards
      - ./infrastructure/grafana/datasources:/etc/grafana/provisioning/datasources
    depends_on:
      - prometheus
    networks:
      - mlops-network
    logging: *default-logging

  # Celery Worker for async tasks
  celery-worker:
    build:
      context: .
      dockerfile: Dockerfile
    container_name: mlops-celery-worker
    environment:
      CELERY_BROKER_URL: redis://:${REDIS_PASSWORD}@redis:6379/0
      CELERY_RESULT_BACKEND: redis://:${REDIS_PASSWORD}@redis:6379/0
      MLFLOW_TRACKING_URI: http://mlflow-tracking:5000
    command: celery -A src.tasks.celery_app worker --loglevel=info
    volumes:
      - ./src:/app/src
    depends_on:
      - redis
      - mlflow-tracking
    networks:
      - mlops-network
    logging: *default-logging

  # Flower for monitoring Celery
  flower:
    image: mher/flower:latest
    container_name: mlops-flower
    environment:
      CELERY_BROKER_URL: redis://:${REDIS_PASSWORD}@redis:6379/0
      FLOWER_PORT: 5555
    ports:
      - "5555:5555"
    depends_on:
      - celery-worker
    networks:
      - mlops-network
    logging: *default-logging

networks:
  mlops-network:
    driver: bridge
    name: mlops-network

volumes:
  postgres_data:
  redis_data:
  prometheus_data:
  grafana_data: