From c839bd2f61a9bcc841c65f1b4d70cf98f7dcf891 Mon Sep 17 00:00:00 2001 From: Anonymoussaurus <50231698+AnonymousWP@users.noreply.github.com> Date: Mon, 4 Nov 2024 17:53:18 +0100 Subject: [PATCH] build(docker): refactor detection of OS and add support for RHEL distros (#211) * build(docker): refactor detection of OS This commit also adds support for RHEL distros * build(install): use functions for installing dependencies * feat(install): enhance installation script with Docker Compose version check and refactor package installation - Refactored the installation script to include a Docker Compose version check, ensuring compatibility with version 2.0.0 or higher. - Consolidated package installation functions into a generic install_package function to streamline the installation process for various packages like nano, curl, and make. - Improved error handling and logging for package installations and Docker setup. * feat(install): enhance Docker and Docker Compose checks in installation script - Added functions to check Docker and Docker Compose installations, including version checks and guidance for installation or upgrade if necessary. - Removed automatic installation of Docker and Docker Compose, replacing it with user guidance for manual installation. - Updated the installation script to ensure Docker is installed and running before proceeding. * fix: improve Docker installation checks and update Makefile - Enhanced the Docker installation script to repeatedly prompt the user until a valid choice is made. - Updated the Makefile to include checks for Docker Compose installation and user permissions, ensuring the user is in the docker group or running as root. * build(install): move pre-req check after env check * build(install): specify color meaning and change some * build(install): provide more specific error information Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> * build(make): use `getent` --------- Co-authored-by: psyray Co-authored-by: sourcery-ai[bot] <58596630+sourcery-ai[bot]@users.noreply.github.com> --- Makefile | 22 +- install.sh | 415 +++++++++++++++++++++--------------- scripts/common_functions.sh | 10 +- 3 files changed, 261 insertions(+), 186 deletions(-) diff --git a/Makefile b/Makefile index 21bc683c..8f22245e 100644 --- a/Makefile +++ b/Makefile @@ -16,9 +16,21 @@ COMPOSE_FILE_DEV := docker/docker-compose.dev.yml COMPOSE_FILE_SETUP := docker/docker-compose.setup.yml SERVICES := db web proxy redis celery celery-beat ollama -# Check if 'docker compose' command is available, otherwise use 'docker-compose' -DOCKER_COMPOSE := $(shell if command -v docker > /dev/null && docker compose version > /dev/null 2>&1; then echo "docker compose"; else echo "docker-compose"; fi) -$(info Using: $(shell echo "$(DOCKER_COMPOSE)")) +# Check if 'docker compose' command is available, otherwise check for 'docker-compose' +DOCKER_COMPOSE := $(shell if command -v docker > /dev/null && docker compose version > /dev/null 2>&1; then echo "docker compose"; elif command -v docker-compose > /dev/null; then echo "docker-compose"; else echo ""; fi) + +ifeq ($(DOCKER_COMPOSE),) +$(error Docker Compose not found. Please install Docker Compose) +endif + +# Check if user is in docker group or is root +DOCKER_GROUP_CHECK := $(shell if [ -n "$$(getent group docker)" ]; then echo "yes"; else echo "no"; fi) + +ifeq ($(DOCKER_GROUP_CHECK),no) +$(error This command must be run with sudo or by a user in the docker group) +endif + +$(info Using: $(DOCKER_COMPOSE)) # Define common commands DOCKER_COMPOSE_CMD := ${COMPOSE_PREFIX_CMD} ${DOCKER_COMPOSE} @@ -150,10 +162,6 @@ help: ## Show this help. @echo " make (default: help)" @echo "" @echo "Targets:" - @awk 'BEGIN {FS = ":.*##"; printf " \033[36m%-15s\033[0m %s\n", "Target", "Description"}' $(MAKEFILE_LIST) - @awk 'BEGIN {FS = ":.*##"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 }' $(MAKEFILE_LIST) - @echo "" - @echo "Special commands:" @echo " make restart [service1] [service2] ... Restart specific services in production mode" @echo " make restart DEV=1 [service1] [service2] ... Restart specific services in development mode" @echo " make restart Restart all services in production mode" diff --git a/install.sh b/install.sh index 38de5a72..1f406e8a 100755 --- a/install.sh +++ b/install.sh @@ -1,9 +1,9 @@ #!/bin/bash # Import common functions -source "$(pwd)/scripts/common_functions.sh" +source "$(pwd)/scripts/common_functions.sh" # Open the file if you want to know the meaning of each color -# Fetch the internal and external IP address so that it can be printed later when the script has finished installing reNgine-ng +# Fetch the internal and external IP address external_ip=$(curl -s https://ipecho.net/plain) internal_ips=$(ip -4 -br addr | awk '$2 == "UP" {print $3} /^lo/ {print $3}' | cut -d'/' -f1) formatted_ips="" @@ -11,6 +11,138 @@ for ip in $internal_ips; do formatted_ips="${formatted_ips}https://$ip\n" done +# Check Docker installation +check_docker_installation() { + while true; do + log "Docker is not installed. You have two options for installation:" $COLOR_CYAN + log "1) Docker Desktop: A user-friendly application with a GUI, suitable for developers. It includes Docker Engine, Docker CLI, Docker Compose, and other tools." $COLOR_GREEN + log "2) Docker Engine: A lightweight, command-line interface suitable for servers and advanced users. It's the core of Docker without additional GUI tools." $COLOR_GREEN + + read -p "Enter your choice (1 or 2): " docker_choice + + case $docker_choice in + 1) + log "Please install Docker Desktop from: https://docs.docker.com/desktop/" $COLOR_YELLOW + break + ;; + 2) + log "Please install Docker Engine from: https://docs.docker.com/engine/install/" $COLOR_YELLOW + break + ;; + *) + log "Invalid choice. Please choose 1 or 2." $COLOR_RED + ;; + esac + done + + log "After installation, please restart this script." $COLOR_CYAN + exit 1 +} + +# Check Docker version and status +check_docker() { + local min_version="20.10.0" + log "Checking Docker installation (minimum required version: $min_version)..." $COLOR_CYAN + + if ! command -v docker &> /dev/null; then + check_docker_installation + fi + + if ! DOCKER_ERROR=$(docker info 2>&1); then + echo "Docker check failed: ${DOCKER_ERROR}" + log "Docker is not running. Please start Docker and try again." $COLOR_RED + log "You can start Docker using: sudo systemctl start docker (on most Linux systems)" $COLOR_YELLOW + exit 1 + fi + + local version=$(docker version --format '{{.Server.Version}}') + + if ! [[ "$(printf '%s\n' "$min_version" "$version" | sort -V | head -n1)" = "$min_version" ]]; then + log "Docker version $version is installed, but reNgine-ng requires version $min_version or higher." $COLOR_RED + log "Please upgrade Docker to continue. Visit https://docs.docker.com/engine/install/ for installation instructions." $COLOR_YELLOW + exit 1 + fi + + log "Docker version $version is installed and running." $COLOR_GREEN + log "It's recommended to use the latest version of Docker. Check https://docs.docker.com/engine/release-notes/ for updates." $COLOR_YELLOW +} + +# Check Docker Compose version and set the appropriate command +check_docker_compose() { + local min_version="2.2.0" + log "Checking Docker Compose installation (minimum required version: $min_version)..." $COLOR_CYAN + + if command -v docker &> /dev/null && docker compose version &> /dev/null; then + DOCKER_COMPOSE="docker compose" + elif command -v docker-compose &> /dev/null; then + DOCKER_COMPOSE="docker-compose" + else + if docker compose version 2>&1 | grep -q "is not a docker command"; then + log "Docker Compose is not installed. Please install Docker Compose v$min_version or later from https://docs.docker.com/compose/install/" $COLOR_RED + log "After installation, please restart this script." $COLOR_CYAN + exit 1 + else + log "An unexpected error occurred while checking for Docker Compose. Please ensure Docker and Docker Compose are correctly installed." $COLOR_RED + exit 1 + fi + fi + + local version=$($DOCKER_COMPOSE version --short) + + if ! [[ "$(printf '%s\n' "$min_version" "$version" | sort -V | head -n1)" = "$min_version" ]]; then + log "Docker Compose version $version is installed, but reNgine-ng requires version $min_version or higher." $COLOR_RED + log "Please upgrade Docker Compose to continue. Visit https://docs.docker.com/compose/install/ for installation instructions." $COLOR_YELLOW + log "After upgrade, please restart this script." $COLOR_CYAN + exit 1 + fi + + log "Using Docker Compose command: $DOCKER_COMPOSE (version $version)" $COLOR_GREEN + log "It's recommended to use the latest version of Docker Compose. Check https://docs.docker.com/compose/release-notes/ for updates." $COLOR_YELLOW + export DOCKER_COMPOSE +} + +# Generic function to install a package +install_package() { + local package_name="$1" + log "Installing $package_name..." $COLOR_CYAN + if ! command -v "$package_name" &> /dev/null; then + . /etc/os-release + DISTRO_FAMILY="${ID_LIKE:-$ID}" + case "$DISTRO_FAMILY" in + *debian*) sudo apt update && sudo apt install -y "$package_name" ;; + *fedora*|*centos*|*rhel*) sudo dnf install -y "$package_name" ;; + *arch*) sudo pacman -Sy "$package_name" ;; + *suse*|*opensuse*) sudo zypper install -y "$package_name" ;; + *) log "Unsupported Linux distribution: $DISTRO_FAMILY. Please install $package_name manually." $COLOR_RED; return 1 ;; + esac + if [ $? -eq 0 ]; then + log "$package_name installed successfully!" $COLOR_GREEN + else + log "Failed to install $package_name. Please check your internet connection and try again." $COLOR_RED + log "If the problem persists, try installing $package_name manually." $COLOR_YELLOW + return 1 + fi + else + log "$package_name is already installed, skipping." $COLOR_GREEN + fi +} + +# Install nano text editor +install_nano() { + install_package "nano" +} + +# Install curl for downloading files +install_curl() { + install_package "curl" +} + +# Install make for building projects +install_make() { + install_package "make" +} + + # Check for root privileges if [ $EUID -eq 0 ]; then if [ "$SUDO_USER" = "root" ] || [ "$SUDO_USER" = "" ]; then @@ -48,190 +180,125 @@ usageFunction() exit 1 } -cat web/art/reNgine.txt - -log "\r\nBefore running this script, please make sure Docker is running and you have made changes to the '.env' file." $COLOR_RED -log "Changing the PostgreSQL username & password in the '.env' is highly recommended.\r\n" $COLOR_RED - -log "Please note that this installation script is only intended for Linux" $COLOR_RED -log "x86_64 and arm64 platform (compatible with Apple Mx series) are supported" $COLOR_RED - -log "Raspbery Pi is not recommended, all install tests have failed" $COLOR_RED -log "" -tput setaf 1; - -isNonInteractive=false -while getopts nh opt; do - case $opt in - n) isNonInteractive=true ;; - h) usageFunction ;; - ?) usageFunction ;; - esac -done - -# Interactive install -if [ $isNonInteractive = false ]; then - read -p "Are you sure you made changes to the '.env' file (y/n)? " answer - case ${answer:0:1} in - y|Y|yes|YES|Yes ) - log "\nContinuing installation!\n" $COLOR_GREEN - ;; - * ) - if ! command -v nano &> /dev/null; then - . /etc/os-release - case "$ID" in - ubuntu|debian) sudo apt update && sudo apt install -y nano ;; - fedora) sudo dnf install -y nano ;; - centos|rhel) sudo yum install -y nano ;; - arch) sudo pacman -Sy nano ;; - opensuse|suse) sudo zypper install -y nano ;; - *) log "Unsupported Linux distribution. Please install nano manually." $COLOR_RED; exit 1 ;; - esac - [ $? -eq 0 ] && log "nano installed!" $COLOR_GREEN || { log "Failed to install nano." $COLOR_RED; exit 1; } - else - log "nano already installed, skipping." $COLOR_GREEN - fi - nano .env - ;; - esac - # Select install type - log "Do you want to build Docker images from source or use pre-built images (recommended)? \nThis saves significant build time but requires good download speeds for it to complete fast." $COLOR_RED - log "1) From source" $COLOR_GREEN - log "2) Use pre-built images (default)" $COLOR_GREEN - read -p "Enter your choice (1 or 2, default is 2): " choice - - case $choice in - 1) - INSTALL_TYPE="source" - ;; - 2|"") - INSTALL_TYPE="prebuilt" - ;; - *) - log "Invalid choice. Defaulting to pre-built images." $COLOR_YELLOW - INSTALL_TYPE="prebuilt" - ;; - esac - - log "Selected installation type: $INSTALL_TYPE" $COLOR_CYAN -fi +# Main installation process +main() { + cat web/art/reNgine.txt + + log "\r\nBefore running this script, please make sure Docker is installed and running, and you have made changes to the '.env' file." $COLOR_RED + log "Changing the PostgreSQL username & password in the '.env' is highly recommended.\r\n" $COLOR_RED + + log "Please note that this installation script is only intended for Linux" $COLOR_RED + log "x86_64 and arm64 platform (compatible with Apple Mx series) are supported" $COLOR_RED + + log "Raspberry Pi is not recommended, all install tests have failed" $COLOR_RED + log "" + tput setaf 1; + + isNonInteractive=false + while getopts nh opt; do + case $opt in + n) isNonInteractive=true ;; + h) usageFunction ;; + ?) usageFunction ;; + esac + done + + if [ $isNonInteractive = false ]; then + read -p "Are you sure you made changes to the '.env' file (y/n)? " answer + case ${answer:0:1} in + y|Y|yes|YES|Yes ) + log "\nContinuing installation!\n" $COLOR_GREEN + ;; + * ) + install_nano + nano .env + ;; + esac + + log "Checking and installing reNgine-ng prerequisites..." $COLOR_CYAN + + install_curl + install_make + check_docker + check_docker_compose + + log "Do you want to build Docker images from source or use pre-built images (recommended)? \nThis saves significant build time but requires good download speeds for it to complete fast." $COLOR_RED + log "1) From source" $COLOR_YELLOW + log "2) Use pre-built images (default)" $COLOR_YELLOW + read -p "Enter your choice (1 or 2, default is 2): " choice + + case $choice in + 1) + INSTALL_TYPE="source" + ;; + 2|"") + INSTALL_TYPE="prebuilt" + ;; + *) + log "Invalid choice. Defaulting to pre-built images." $COLOR_RED + INSTALL_TYPE="prebuilt" + ;; + esac + + log "Selected installation type: $INSTALL_TYPE" $COLOR_CYAN + fi -# Non interactive install -if [ $isNonInteractive = true ]; then - # Check if .env file exists and load vars from env file - if [ -f .env ]; then - export $(grep -v '^#' .env | xargs) - else - log "Error: .env file not found, copy/paste the .env-dist file to .env and edit it" $COLOR_RED + # Non-interactive install + if [ $isNonInteractive = true ]; then + # Load and verify .env file + if [ -f .env ]; then + export $(grep -v '^#' .env | xargs) + else + log "Error: .env file not found, copy/paste the .env-dist file to .env and edit it" $COLOR_RED + exit 1 + fi + + if [ -z "$DJANGO_SUPERUSER_USERNAME" ] || [ -z "$DJANGO_SUPERUSER_EMAIL" ] || [ -z "$DJANGO_SUPERUSER_PASSWORD" ]; then + log "Error: DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD must be set in .env for non-interactive installation" $COLOR_RED exit 1 - fi + fi - if [ -z "$DJANGO_SUPERUSER_USERNAME" ] || [ -z "$DJANGO_SUPERUSER_EMAIL" ] || [ -z "$DJANGO_SUPERUSER_PASSWORD" ]; then - log "Error: DJANGO_SUPERUSER_USERNAME, DJANGO_SUPERUSER_EMAIL, and DJANGO_SUPERUSER_PASSWORD must be set in .env for non-interactive installation" $COLOR_RED - exit 1 + INSTALL_TYPE=${INSTALL_TYPE:-prebuilt} + log "Non-interactive installation parameter set. Installation begins." $COLOR_GREEN fi - # Define INSTALL_TYPE from .env or use a default value + if [ -z "$INSTALL_TYPE" ]; then - log "Warning: INSTALL_TYPE is not set in .env for non-interactive installation, fallback to prebuilt install" $COLOR_YELLOW + log "Error: INSTALL_TYPE is not set" $COLOR_RED + exit 1 + elif [ "$INSTALL_TYPE" != "prebuilt" ] && [ "$INSTALL_TYPE" != "source" ]; then + log "Error: INSTALL_TYPE must be either 'prebuilt' or 'source'" $COLOR_RED + exit 1 fi - INSTALL_TYPE=${INSTALL_TYPE:-prebuilt} - log "Non-interactive installation parameter set. Installation begins." $COLOR_GREEN -fi -log "Installing reNgine-ng and its dependencies..." $COLOR_CYAN - -log "Installing curl..." $COLOR_CYAN - -if ! command -v curl &> /dev/null; then - . /etc/os-release - case "$ID" in - ubuntu|debian) sudo apt update && sudo apt install -y curl ;; - fedora) sudo dnf install -y curl ;; - centos|rhel) sudo yum install -y curl ;; - arch) sudo pacman -Sy curl ;; - opensuse|suse) sudo zypper install -y curl ;; - *) log "Unsupported Linux distribution. Please install curl manually." $COLOR_RED; exit 1 ;; - esac - [ $? -eq 0 ] && log "CURL installed!" $COLOR_GREEN || { log "Failed to install CURL." $COLOR_RED; exit 1; } -else - log "CURL already installed, skipping." $COLOR_GREEN -fi - -log "Installing Docker..." $COLOR_CYAN -if ! command -v docker 2> /dev/null; then - curl -fsSL https://get.docker.com -o get-docker.sh && sh get-docker.sh - log "Docker installed!" $COLOR_GREEN -else - log "Docker already installed, skipping." $COLOR_GREEN -fi - -log "Installing Docker Compose..." $COLOR_CYAN -if ! command -v docker compose 2> /dev/null; then - curl -L "https://github.com/docker/compose/releases/latest/download/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose - ln -s /usr/local/bin/docker-compose /usr/bin/docker-compose - log "Docker Compose installed!" $COLOR_GREEN -else - log "Docker Compose already installed, skipping." $COLOR_GREEN -fi + log "Installing reNgine-ng from $INSTALL_TYPE, please be patient as the installation could take a while..." $COLOR_CYAN + sleep 5 -if ! command -v make &> /dev/null; then - . /etc/os-release - case "$ID" in - ubuntu|debian) sudo apt update && sudo apt install -y make ;; - fedora) sudo dnf install -y make ;; - centos|rhel) sudo yum install -y make ;; - arch) sudo pacman -Sy make ;; - opensuse|suse) sudo zypper install -y make ;; - *) log "Unsupported Linux distribution. Please install make manually." $COLOR_RED; exit 1 ;; - esac - [ $? -eq 0 ] && log "make installed!" $COLOR_GREEN || { log "Failed to install make." $COLOR_RED; exit 1; } -else - log "make already installed, skipping." $COLOR_GREEN -fi + log "Generating certificates..." $COLOR_CYAN + make certs && log "Certificates have been generated" $COLOR_GREEN || { log "Certificate generation failed!" $COLOR_RED; exit 1; } -log "Checking Docker status..." $COLOR_CYAN -if docker info >/dev/null 2>&1; then - log "Docker is running." $COLOR_GREEN -else - log "Docker is not running. Please run Docker and try again." $COLOR_RED - log "You can run Docker service using: sudo systemctl start docker" $COLOR_RED - exit 1 -fi - -if [ -z "$INSTALL_TYPE" ]; then - log "Error: INSTALL_TYPE is not set" $COLOR_RED - exit 1 -elif [ "$INSTALL_TYPE" != "prebuilt" ] && [ "$INSTALL_TYPE" != "source" ]; then - log "Error: INSTALL_TYPE must be either 'prebuilt' or 'source'" $COLOR_RED - exit 1 -fi - -log "Installing reNgine-ng from $INSTALL_TYPE, please be patient as the installation could take a while..." $COLOR_CYAN -sleep 5 - -log "Generating certificates..." $COLOR_CYAN -make certs && log "Certificates have been generated" $COLOR_GREEN || { log "Certificate generation failed!" $COLOR_RED; exit 1; } + if [ "$INSTALL_TYPE" = "source" ]; then + log "Building Docker images..." $COLOR_CYAN + make build && log "Docker images have been built" $COLOR_GREEN || { log "Docker images build failed!" $COLOR_RED; exit 1; } + fi -if [ "$INSTALL_TYPE" = "source" ]; then - log "Building Docker images..." $COLOR_CYAN - make build && log "Docker images have been built" $COLOR_GREEN || { log "Docker images build failed!" $COLOR_RED; exit 1; } -fi + if [ "$INSTALL_TYPE" = "prebuilt" ]; then + log "Pulling pre-built Docker images..." $COLOR_CYAN + make pull && log "Docker images have been pulled" $COLOR_GREEN || { log "Docker images pull failed!" $COLOR_RED; exit 1; } + fi -if [ "$INSTALL_TYPE" = "prebuilt" ]; then - log "Pulling pre-built Docker images..." $COLOR_CYAN - make pull && log "Docker images have been pulled" $COLOR_GREEN || { log "Docker images pull failed!" $COLOR_RED; exit 1; } -fi + log "Docker containers starting, please wait as starting the Celery container could take a while..." $COLOR_CYAN + sleep 5 + make up && log "reNgine-ng is started!" $COLOR_GREEN || { log "reNgine-ng start failed!" $COLOR_RED; exit 1; } -log "Docker containers starting, please wait as starting the Celery container could take a while..." $COLOR_CYAN -sleep 5 -make up && log "reNgine-ng is started!" $COLOR_GREEN || { log "reNgine-ng start failed!" $COLOR_RED; exit 1; } + log "Creating an account..." $COLOR_CYAN + make superuser_create isNonInteractive=$isNonInteractive -log "Creating an account..." $COLOR_CYAN -make superuser_create isNonInteractive=$isNonInteractive + log "reNgine-ng is successfully installed and started!" $COLOR_GREEN + log "\r\nThank you for installing reNgine-ng, happy recon!" $COLOR_GREEN -log "reNgine-ng is successfully installed and started!" $COLOR_GREEN -log "\r\nThank you for installing reNgine-ng, happy recon!" $COLOR_GREEN + log "\r\nIn case you're running this locally, reNgine-ng should be available at one of the following IPs:\n$formatted_ips" $COLOR_GREEN + log "In case you're running this on a server, reNgine-ng should be available at: https://$external_ip/" $COLOR_GREEN +} -log "\r\nIn case you're running this locally, reNgine-ng should be available at one of the following IPs:\n$formatted_ips" $COLOR_GREEN -log "In case you're running this on a server, reNgine-ng should be available at: https://$external_ip/" $COLOR_GREEN +# Run the main installation process +main diff --git a/scripts/common_functions.sh b/scripts/common_functions.sh index 3fec890d..a3d4566f 100644 --- a/scripts/common_functions.sh +++ b/scripts/common_functions.sh @@ -2,13 +2,13 @@ # Define color codes. COLOR_BLACK=0 -COLOR_RED=1 -COLOR_GREEN=2 -COLOR_YELLOW=3 +COLOR_RED=1 # For errors and important messages +COLOR_GREEN=2 # For succesful output/messages +COLOR_YELLOW=3 # For questions and choices COLOR_BLUE=4 COLOR_MAGENTA=5 -COLOR_CYAN=6 -COLOR_WHITE=7 +COLOR_CYAN=6 # For actions that are being executed +COLOR_WHITE=7 # Default, we don't really use this explicitly COLOR_DEFAULT=$COLOR_WHITE # Use white as default for clarity # Log messages in different colors