echostack-org · leewujung · Jun 5, 2026 · Jun 5, 2026 · Jun 5, 2026 · Jun 5, 2026
diff --git a/README.md b/README.md
@@ -33,7 +33,7 @@ Echodataflow streamlines echosounder data processing by combining [Prefect](http
    clone the repo and install it like below:
    ```bash
    git clone git+https://github.com/echostack-org/echodataflow.git  # clone the repo
-   pip install -e .[test,lint,docs]  # install in editable mode with dev tools
+   pip install -e ".[test,lint,docs]"  # install in editable mode with dev tools
    ```
 
 
@@ -61,7 +61,7 @@ Echodataflow streamlines echosounder data processing by combining [Prefect](http
    python -m echodataflow.deployment.deploy_cli run \
    --source-mode local \
    --default-work-pool-name local \
-   --param-config REPO_DIRECTORY/recipes/params/config_{MISSION_NAME}.yaml \
+   --param-config REPO_DIRECTORY/recipes/params/params_{MISSION_NAME}.yaml \
    --deploy-spec REPO_DIRECTORY/recipes/deploy/deploy_{MISSION_NAME}.yaml
    ```
 
@@ -80,14 +80,98 @@ Echodataflow streamlines echosounder data processing by combining [Prefect](http
    git clone https://github.com/echostack-org/echodataflow-recipes.git
    ```
 
-5. Deploy and run the ship pipeline:
+5. Deploy and run the cloud pipeline:
    ```bash
-   python -m echodataflow.deployment.deploy_cli run --default-work-pool-name local --param-config REPO_DIRECTORY/recipes/params/config_cloud_2025.yaml --deploy-spec REPO_DIRECTORY/recipes/deploy/deploy_cloud_2025.yaml --source-mode local
+   python -m echodataflow.deployment.deploy_cli run \
+   --source-mode local \
+   --default-work-pool-name local \
+   --param-config REPO_DIRECTORY/recipes/params/params_{MISSION_NAME}.yaml \
+   --deploy-spec REPO_DIRECTORY/recipes/deploy/deploy_{MISSION_NAME}.yaml
    ```
 
 6. Start up system services that hosts the 2 sets of visualization
 
 
+## Running Local Prefect Services on macOS (launchd)
+
+To run a local Prefect server and worker as background services on macOS, you can
+use launchd with the provided plist templates:
+
+- `src/echodataflow/services/deploy_prefect_server.launchd.plist`
+- `src/echodataflow/services/deploy_prefect_worker.launchd.plist`
+
+These templates intentionally use direct one-line `ProgramArguments` commands, similar
+to `.service` `ExecStart` usage, with no wrapper shell script required.
+
+1. Copy and customize the templates for your user:
+   ```shell
+   mkdir -p ~/.config/echodataflow ~/Library/LaunchAgents ~/.local/var/log/echodataflow
+   cp src/echodataflow/services/services.env.example ~/.config/echodataflow/services.env
+   cp src/echodataflow/services/deploy_prefect_server.launchd.plist ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist
+   cp src/echodataflow/services/deploy_prefect_worker.launchd.plist ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist
+   ```
+
+2. Edit `~/.config/echodataflow/services.env` as needed:
+   - Adjust `ECHODATAFLOW_ENV`
+   - Adjust `ECHODATAFLOW_HOME`
+   - Adjust `MAMBA_BIN`
+   - Adjust `PREFECT_POOL`
+   - Adjust `PREFECT_API_URL`
+
+3. Load and start services:
+   ```shell
+   launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist
+   launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist
+   launchctl kickstart -k gui/$(id -u)/org.echodataflow.prefect-server
+   launchctl kickstart -k gui/$(id -u)/org.echodataflow.prefect-worker
+   ```
+
+4. Check status and logs:
+   ```shell
+   # make sure "state = running" and "runs" not increasing
+   launchctl print gui/$(id -u)/org.echodataflow.prefect-server
+   launchctl print gui/$(id -u)/org.echodataflow.prefect-worker
+   # -f to follow logs in real time
+   tail -n 100 ~/.local/var/log/echodataflow/prefect-server.err.log
+   tail -n 100 ~/.local/var/log/echodataflow/prefect-worker.err.log
+   ```
+
+5. To stop and unload services:
+   ```shell
+   launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist
+   launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist
+   ```
+
+6. SQLite health checks (local Prefect server):
+   ```shell
+   sqlite3 ~/.prefect/prefect.db "PRAGMA quick_check;"
+   sqlite3 ~/.prefect/prefect.db "PRAGMA integrity_check;"
+   ```
+
+7. If server startup keeps failing with SQLite lock errors, reset local DB safely:
+   ```shell
+   # stop services first
+   launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist
+   launchctl bootout gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist
+
+   # archive existing local Prefect DB files (do not delete first)
+   ts=$(date +%Y%m%d_%H%M%S)
+   mkdir -p ~/.prefect/db-backups/$ts
+   mv ~/.prefect/prefect.db* ~/.prefect/db-backups/$ts/ 2>/dev/null || true
+
+   # start server, then worker
+   launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-server.plist
+   launchctl kickstart -k gui/$(id -u)/org.echodataflow.prefect-server
+   launchctl bootstrap gui/$(id -u) ~/Library/LaunchAgents/org.echodataflow.prefect-worker.plist
+   launchctl kickstart -k gui/$(id -u)/org.echodataflow.prefect-worker
+   ```
+
+Notes:
+- `ThrottleInterval=30` in plist files helps avoid aggressive restart loops.
+- `database is locked` usually means SQLite write contention, not corruption.
+- For heavier multi-flow usage, move Prefect server DB to Postgres.
+
+
 
 ## License
 

diff --git a/src/echodataflow/services/deploy_prefect_server.launchd.plist b/src/echodataflow/services/deploy_prefect_server.launchd.plist
@@ -0,0 +1,68 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>org.echodataflow.prefect-server</string>
+
+  <key>RunAtLoad</key>
+  <true/>
+
+  <key>KeepAlive</key>
+  <true/>
+
+  <key>ThrottleInterval</key>
+  <integer>30</integer>
+
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string>
+  </dict>
+
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/zsh</string>
+    <string>-lc</string>
+    <string>
+ENV_FILE="${ECHODATAFLOW_SERVICES_ENV_FILE:-$HOME/.config/echodataflow/services.env}"
+[[ -f "$ENV_FILE" ]] || { echo "Missing $ENV_FILE"; exit 1; }
+
+set -a
+source "$ENV_FILE"
+set +a
+
+: "${ECHODATAFLOW_HOME:?ECHODATAFLOW_HOME is not set in services.env}"
+: "${ECHODATAFLOW_WORKDIR:?ECHODATAFLOW_WORKDIR is not set in services.env}"
+: "${ECHODATAFLOW_LOG_DIR:?ECHODATAFLOW_LOG_DIR is not set in services.env}"
+
+mkdir -p "$ECHODATAFLOW_LOG_DIR"
+cd "$ECHODATAFLOW_WORKDIR" || { echo "Missing WORKDIR: $ECHODATAFLOW_WORKDIR"; exit 1; }
+
+exec >>"$ECHODATAFLOW_LOG_DIR/prefect-server.out.log" 2>>"$ECHODATAFLOW_LOG_DIR/prefect-server.err.log"
+
+DB_PATH="${PREFECT_DB_PATH:-$ECHODATAFLOW_HOME/.prefect/prefect.db}"
+AUTO_RESET="${PREFECT_DB_AUTORESET_ON_FAILURE:-false}"
+
+if [[ -f "$DB_PATH" ]]; then
+  DB_CHECK="$(sqlite3 "$DB_PATH" "PRAGMA quick_check;" 2>/dev/null || true)"
+  if [[ "$DB_CHECK" != "ok" ]]; then
+    echo "Prefect DB health check failed for $DB_PATH: ${DB_CHECK:-no output}"
+    if [[ "$AUTO_RESET" == "true" ]]; then
+      ts="$(date +%Y%m%d_%H%M%S)"
+      backup_dir="$ECHODATAFLOW_HOME/.prefect/db-backups/$ts"
+      mkdir -p "$backup_dir"
+      mv "${DB_PATH}"* "$backup_dir"/ 2>/dev/null || true
+      echo "Archived DB files to $backup_dir"
+    else
+      echo "Set PREFECT_DB_AUTORESET_ON_FAILURE=true in services.env to auto-archive and reset."
+      exit 1
+    fi
+  fi
+fi
+
+exec "$MAMBA_BIN" run -n "$ECHODATAFLOW_ENV" prefect server start --host 127.0.0.1 --port 4200
+    </string>
+  </array>
+</dict>
+</plist>
diff --git a/src/echodataflow/services/deploy_prefect_worker.launchd.plist b/src/echodataflow/services/deploy_prefect_worker.launchd.plist
@@ -0,0 +1,48 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key>
+  <string>org.echodataflow.prefect-worker</string>
+
+  <key>RunAtLoad</key>
+  <true/>
+
+  <key>KeepAlive</key>
+  <true/>
+
+  <key>ThrottleInterval</key>
+  <integer>30</integer>
+
+  <key>EnvironmentVariables</key>
+  <dict>
+    <key>PATH</key>
+    <string>/opt/homebrew/bin:/usr/local/bin:/usr/bin:/bin</string>
+  </dict>
+
+  <key>ProgramArguments</key>
+  <array>
+    <string>/bin/zsh</string>
+    <string>-lc</string>
+    <string>
+ENV_FILE="${ECHODATAFLOW_SERVICES_ENV_FILE:-$HOME/.config/echodataflow/services.env}"
+[[ -f "$ENV_FILE" ]] || { echo "Missing $ENV_FILE"; exit 1; }
+
+set -a
+source "$ENV_FILE"
+set +a
+
+: "${ECHODATAFLOW_HOME:?ECHODATAFLOW_HOME is not set in services.env}"
+: "${ECHODATAFLOW_WORKDIR:?ECHODATAFLOW_WORKDIR is not set in services.env}"
+: "${ECHODATAFLOW_LOG_DIR:?ECHODATAFLOW_LOG_DIR is not set in services.env}"
+
+mkdir -p "$ECHODATAFLOW_LOG_DIR"
+cd "$ECHODATAFLOW_WORKDIR" || { echo "Missing WORKDIR: $ECHODATAFLOW_WORKDIR"; exit 1; }
+
+exec >>"$ECHODATAFLOW_LOG_DIR/prefect-worker.out.log" 2>>"$ECHODATAFLOW_LOG_DIR/prefect-worker.err.log"
+
+exec "$MAMBA_BIN" run -n "$ECHODATAFLOW_ENV" prefect worker start --pool "$PREFECT_POOL"
+    </string>
+  </array>
+</dict>
+</plist>
diff --git a/src/echodataflow/services/services.env.example b/src/echodataflow/services/services.env.example
diff --git a/src/echodataflow/services/services.env.example_cloud b/src/echodataflow/services/services.env.example_cloud
@@ -0,0 +1,31 @@
+# Copy this file to: ~/.config/echodataflow/services.env
+# Then adjust values as needed.
+
+# Name of conda environment to run services in
+ECHODATAFLOW_ENV=echodataflow_20260604
+
+# Base home directory used by launchd service scripts
+ECHODATAFLOW_HOME=/Users/exouser
+
+# Project working directory used by launchd service scripts
+ECHODATAFLOW_WORKDIR=$ECHODATAFLOW_HOME/code_git/echodataflow
+
+# Log directory used by launchd service scripts
+ECHODATAFLOW_LOG_DIR=$ECHODATAFLOW_HOME/.local/var/log/echodataflow
+
+# Full path to conda or mamba executable
+MAMBA_BIN=$ECHODATAFLOW_HOME/miniforge3/bin/mamba
+
+# Prefect worker pool name
+PREFECT_POOL=local
+
+# Prefect API URL used by local workers
+PREFECT_API_URL=http://127.0.0.1:4200/api
+
+# Optional: Prefect local SQLite DB file path for server preflight checks
+PREFECT_DB_PATH=$ECHODATAFLOW_HOME/.prefect/prefect.db
+
+# Optional safety switch for server startup:
+# - false: fail fast if DB quick_check is not ok
+# - true: archive old DB files and start with a new DB
+PREFECT_DB_AUTORESET_ON_FAILURE=true
diff --git a/src/echodataflow/services/services.env.example_local b/src/echodataflow/services/services.env.example_local
@@ -0,0 +1,31 @@
+# Copy this file to: ~/.config/echodataflow/services.env
+# Then adjust values as needed.
+
+# Name of conda environment to run services in
+ECHODATAFLOW_ENV=echodataflow_20260604
+
+# Base home directory used by launchd service scripts
+ECHODATAFLOW_HOME=/Users/feresa
+
+# Project working directory used by launchd service scripts
+ECHODATAFLOW_WORKDIR=$ECHODATAFLOW_HOME/code_git/echodataflow
+
+# Log directory used by launchd service scripts
+ECHODATAFLOW_LOG_DIR=$ECHODATAFLOW_HOME/.local/var/log/echodataflow
+
+# Full path to conda or mamba executable
+MAMBA_BIN=$ECHODATAFLOW_HOME/miniforge3/bin/mamba
+
+# Prefect worker pool name
+PREFECT_POOL=local
+
+# Prefect API URL used by local workers
+PREFECT_API_URL=http://127.0.0.1:4200/api
+
+# Optional: Prefect local SQLite DB file path for server preflight checks
+PREFECT_DB_PATH=$ECHODATAFLOW_HOME/.prefect/prefect.db
+
+# Optional safety switch for server startup:
+# - false: fail fast if DB quick_check is not ok
+# - true: archive old DB files and start with a new DB
+PREFECT_DB_AUTORESET_ON_FAILURE=true