diff --git a/scripts/metaflow_minikube b/scripts/metaflow_minikube new file mode 100755 index 0000000..fa8351b --- /dev/null +++ b/scripts/metaflow_minikube @@ -0,0 +1,503 @@ +#!/bin/bash +set -euo pipefail + +# Usage helper +usage() { + echo "Usage: $0 --install | --cleanup" + exit 1 +} + +# Check that exactly one parameter is provided. +if [ "$#" -ne 1 ]; then + usage +fi + +MODE="$1" + +######################################## +# Helm Repository Setup Function +######################################## +setup_helm_repos() { + echo "Setting up Helm repositories..." + repos=( + "minio-s3=https://charts.min.io/" + "bitnami=https://charts.bitnami.com/bitnami" + "metaflow=https://outerbounds.github.io/metaflow-tools" + "argo=https://argoproj.github.io/argo-helm" + ) + for repo in "${repos[@]}"; do + alias="${repo%%=*}" + url="${repo#*=}" + if helm repo list | awk '{print $1}' | grep -qx "$alias"; then + echo "Helm repo '$alias' already exists." + else + echo "Adding Helm repo '$alias'..." + helm repo add "$alias" "$url" + fi + done + helm repo update +} + +######################################## +# Installation Functions (Installation Mode) +######################################## + +# Step 1: Check required commands and Python pip packages. +check_dependencies() { + echo "Step 1: Checking dependencies..." + missing=false + + # Check required commands + for cmd in minikube kubectl helm; do + if ! command -v "$cmd" >/dev/null 2>&1; then + echo "✖ ERROR: $cmd is not installed or not in PATH." + missing=true + else + echo "✔ $cmd found." + fi + done + + if [ "$missing" = true ]; then + echo "Please install the missing commands and re-run the script." + exit 1 + fi + + # Check pip packages: metaflow and kubernetes + echo "" + echo "Checking Python pip packages..." + for package in metaflow kubernetes; do + if ! python3 -m pip show "$package" >/dev/null 2>&1; then + echo "✖ $package not found. Installing..." + python3 -m pip install "$package" + else + echo "✔ $package is installed." + fi + done +} + +# Step 2: Start Minikube cluster. +start_minikube() { + echo "Step 2: Starting minikube cluster..." + cluster_status=$(minikube status --format "{{.Host}}" 2>/dev/null || echo "Stopped") + if [ "$cluster_status" == "Running" ]; then + echo "Minikube cluster is already running." + else + echo "Starting minikube with 6 CPUs and 10240 MB memory..." + minikube start --cpus 6 --memory 10240 + fi +} + +# Step 3: Deploy MinIO via Helm. +deploy_minio() { + echo "Step 3: Deploying MinIO..." + helm upgrade --install minio-s3 minio-s3/minio \ + --set resources.requests.memory=512Mi \ + --set replicas=1 --set persistence.enabled=false \ + --set mode=standalone \ + --set rootUser=rootuser \ + --set rootPassword=rootpass123 \ + --set buckets[0].name=metaflow-test,buckets[0].policy=none,buckets[0].purge=false +} + +# Step 4: Create Kubernetes secret for MinIO. +create_minio_secret() { + echo "Step 4: Creating MinIO secret..." + kubectl apply -f <(cat < /tmp/minio-portforward.log 2>&1 & + echo "MinIO port-forward running in the background (log: minio-portforward.log)." +} + + + +# Step 9: Install Metaflow metadata service via Helm. +install_metadata_service() { + echo "Step 9: Installing Metaflow metadata service..." + helm upgrade --install metaflow-service metaflow/metaflow-service \ + --timeout 15m0s \ + --set metadatadb.user=metaflow \ + --set metadatadb.password=metaflow \ + --set metadatadb.database=metaflow \ + --set metadatadb.host=postgresql +} + +# Step 10: Port-forward the metadata service. +portforward_metadata() { + echo "Step 10: Port-forwarding metadata service on port 8080..." + echo "Waiting for metadata service pods to be ready..." + kubectl wait --for=condition=ready pod -l app.kubernetes.io/name=metaflow-service --timeout=300s + + METAFLOW_SVC=$(kubectl get svc -l app.kubernetes.io/name=metaflow-service -o jsonpath='{.items[0].metadata.name}') + if [ -z "$METAFLOW_SVC" ]; then + echo "ERROR: Could not locate metaflow-service." + exit 1 + fi + kubectl port-forward svc/"$METAFLOW_SVC" 8080:8080 > /tmp/metadata-service-portforward.log 2>&1 & + echo "Metadata service port-forward running on port 8080 (log: metadata-service-portforward.log)." +} + +# Step 11: Install Argo Workflows. +install_argo() { + echo "Step 11: Installing Argo Workflows..." + kubectl create ns argo || echo "Namespace 'argo' already exists." + kubectl apply -n argo -f https://github.com/argoproj/argo-workflows/releases/latest/download/quick-start-postgres.yaml + kubectl apply -f <(cat < /tmp/metaflow-ui-backend.log 2>&1 & + echo "Metaflow UI backend port-forward running on port 8083 (log: metaflow-ui-backend.log)." + + METAFLOW_UI_STATIC=$(kubectl get svc -l app.kubernetes.io/name=metaflow-ui-static -o jsonpath='{.items[0].metadata.name}') + if [ -z "$METAFLOW_UI_STATIC" ]; then + echo "ERROR: Metaflow UI static service not found." + exit 1 + fi + kubectl port-forward svc/"$METAFLOW_UI_STATIC" 3000:3000 > /tmp/metaflow-ui-static.log 2>&1 & + echo "Metaflow UI static port-forward running on port 3000 (log: metaflow-ui-static.log)." +} + +# Step 16: Create Metaflow configuration file. +create_metaflow_config() { + echo "Step 16: Creating Metaflow config file..." + CONFIG_DIR="$HOME/.metaflowconfig" + mkdir -p "$CONFIG_DIR" + cat > "$CONFIG_DIR/config_minikube.json" <