[ADP-3409] Boostrap mainnet node via mithril (#4733)

- [x] Docker based mithril bootup buildkite step,
https://buildkite.com/cardano-foundation/cardano-wallet/builds/6843#01915ab0-486b-43c2-a963-7f5c215b726e/220-237
- [x] Nix based mithril bootup buildkite step ,
https://buildkite.com/cardano-foundation/cardano-wallet/builds/6843#01915ab0-4828-4fe2-82f2-03ffd856303b/6-23

ADP-3409

Author: paolino

.buildkite/pipeline.yml

@@ -19,6 +19,7 @@ steps:
     # Check whether regenerate.sh was applied when it had to be applied.
     key: linux-nix
     commands:
+      - nix --version
       - nix/regenerate.sh
     agents:
       system: ${linux}
@@ -37,11 +38,11 @@ steps:
   - label: Check Nix (macOS)
     key: macos-nix
     commands:
-      - 'nix flake info'
+        - nix --version
+        - nix flake info
     agents:
       system: ${macos}
 
-
   - label: Dev Shell Attic Cache (macos)
     depends_on:
       - macos-nix
@@ -190,6 +191,33 @@ steps:
         CLEANUP_DB: true
         NETWORK: mainnet
 
+    - block: Mainnet Boot Sync with Mithril
+      if: build.env("RELEASE_CANDIDATE") == null
+      depends_on: []
+      key: linux-mainnet-full-sync-block
+
+    - label: Mainnet Boot Sync with Mithril
+      timeout: 120
+      depends_on:
+        - linux-mainnet-full-sync-block
+      command: |
+        cd run/mainnet/nix
+        rm -rf logs
+        mkdir -p logs
+        rm -rf databases
+        ./run.sh sync
+      artifact_paths:
+        - "./run/mainnet/nix/logs/*"
+      agents:
+        system: x86_64-linux
+      env:
+        SUCCESS_STATUS: syncing
+        NODE_LOGS_FILE: ./logs/node.log
+        WALLET_LOGS_FILE: ./logs/wallet.log
+        CLEANUP_DB: true
+        NETWORK: mainnet
+        USE_MITHRIL: true
+
     - block: Sanchonet Full Sync
       if: build.env("RELEASE_CANDIDATE") == null
       depends_on: []
@@ -481,7 +509,6 @@ steps:
         build.branch !~ /^gh-readonly-queue\/master/
           && build.branch != "master"
           && ( build.env("RELEASE_CANDIDATE") == null || build.env("TEST_RC") == "TRUE" )
-      depends_on: macos-nix
       key: block-macos
 
     - label: Build Integration Tests (macOS, arm64)
@@ -667,6 +694,20 @@ steps:
           SUCCESS_STATUS: syncing
           USE_LOCAL_IMAGE: true
 
+      - label: Mainnet Boot Sync via Mithril
+        timeout_in_minutes: 120
+        command: |
+          cd run/mainnet/docker
+          export WALLET_TAG=$(buildkite-agent meta-data get "release-cabal-version")
+          rm -rf databases
+          ./run.sh node-db-with-mithril
+          ./run.sh sync
+        agents:
+          system: x86_64-linux
+        env:
+          SUCCESS_STATUS: syncing
+          USE_LOCAL_IMAGE: true
+
       - label: Private Network Full Sync
         timeout: 20
         command: |

run/common/docker/run.sh

@@ -1,13 +1,14 @@
 #! /bin/bash
 
 # Enforce strict script execution modes
-set -euo pipefail
+set -euox pipefail
 
 # Function to display usage information
 usage() {
     echo "Usage: $0 [sync|start|stop|logs|help]"
     echo "  sync: Sync the service and wait for it to be ready"
     echo "  start: Start the service"
+    echo "  start-with-mithril: Start the service with mithril boostrap"
     echo "  stop: Stop the service"
     echo "  logs: Show the service logs"
     echo "  help: Show this help message"
@@ -53,11 +54,10 @@ rm -rf "${WALLET_DB:?}/*"
 # Define a local db if NODE_DB is not set
 if [[ -z "${NODE_DB-}" ]]; then
     LOCAL_NODE_DB=./databases/node-db
-    mkdir -p $LOCAL_NODE_DB
     NODE_DB=$LOCAL_NODE_DB
     export NODE_DB
 fi
-rm -rf "${NODE_DB:?}/*"
+mkdir -p "${NODE_DB:?}"
 
 # Get the current user's ID and export it
 USER_ID=$(id -u)
@@ -94,8 +94,20 @@ startup() {
 cleanup() {
     echo "Cleaning up..."
     docker compose down 2>/dev/null
+    sleep  3
+    docker compose kill 2>/dev/null
 }
 
+node-db-with-mithril() {
+    if [ "$NETWORK" != "mainnet" ]; then
+        echo "Error: This option is only available for the mainnet network"
+        exit 1
+    fi
+    echo "Starting the mithril service..."
+    digest=$(docker compose -p mithril run --rm mithril cdb  snapshot list --json | jq -r .[0].digest)
+    rm -rf "${NODE_DB:?}"/*
+    docker compose -p mithril run --rm mithril cdb download "$digest"
+}
 
 # Case statement to handle different command-line arguments
 case "$1" in
@@ -164,6 +176,9 @@ case "$1" in
         echo "Showing logs..."
         docker compose logs -f  # Follow the service logs
         ;;
+    node-db-with-mithril)
+        node-db-with-mithril
+        ;;
     help)
         usage  # Display usage information
         ;;

run/common/nix/run.sh

@@ -1,4 +1,4 @@
-#! /usr/bin/env -S nix shell '.#cardano-wallet' '.#cardano-node' '.#cardano-cli' --command bash
+#! /usr/bin/env -S nix shell '.#cardano-wallet' '.#cardano-node' '.#cardano-cli' 'github:input-output-hk/mithril' --command bash
 # shellcheck shell=bash
 
 # set -euox pipefail
@@ -47,10 +47,6 @@ if [[ -z "${NODE_DB-}" ]]; then
     export NODE_DB
 fi
 
-if [[ -n "${CLEANUP_DB-}" ]]; then
-    rm -rf "${NODE_DB:?}"/*
-fi
-
 NETWORK=${NETWORK:=testnet}
 
 # Define and export the node socket name
@@ -72,17 +68,34 @@ LOCAL_NODE_LOGS_FILE=./node.log
 NODE_LOGS_FILE="${NODE_LOGS_FILE:=$LOCAL_NODE_LOGS_FILE}"
 
 cleanup() {
-    # shellcheck disable=SC2317
     echo "Cleaning up..."
-    # shellcheck disable=SC2317
-    kill "${NODE_ID}" || echo "Failed to kill node"
-    # shellcheck disable=SC2317
-    kill "${WALLET_ID}" || echo "Failed to kill wallet"
+    kill "${NODE_ID-}" || echo "Failed to kill node"
+    kill "${WALLET_ID-}" || echo "Failed to kill wallet"
+    sleep 5
+    if [[ -n "${CLEANUP_DB-}" ]]; then
+        echo "Cleaning up databases..."
+        rm -rf "${NODE_DB:?}"/* || echo "Failed to clean node db"
+        rm -rf "${WALLET_DB:?}"/* || echo "Failed to clean wallet db"
+    fi
 }
 
 # Trap the cleanup function on exit
 trap cleanup ERR INT EXIT
 
+if [[ -n "${USE_MITHRIL-}" ]];
+    then
+        if [ "$NETWORK" != "mainnet" ]; then
+            echo "Error: This option is only available for the mainnet network"
+            exit 1
+        fi
+        echo "Starting the mithril service..."
+        rm -rf "${NODE_DB:?}"/*
+        export AGGREGATOR_ENDPOINT
+        export GENESIS_VERIFICATION_KEY
+        digest=$(mithril-client cdb  snapshot list --json | jq -r .[0].digest)
+        (cd "${NODE_DB}" && mithril-client cdb download "$digest")
+        (cd "${NODE_DB}" && mv db/* . && rmdir db)
+fi
 
 # Start the node with logs redirected to a file if NODE_LOGS_FILE is set
 # shellcheck disable=SC2086
@@ -157,18 +170,11 @@ else
     WALLET_ID=$!
 fi
 
-cleanup() {
-    echo "Cleaning up..."
-    kill "${NODE_ID}" || echo "Failed to kill node"
-    kill "${WALLET_ID}" || echo "Failed to kill wallet"
-}
-
-# Trap the cleanup function on exit
-trap cleanup ERR INT EXIT
 
 # Case statement to handle different command-line arguments
 case "$1" in
     sync)
+
         echo "Wallet service port: $WALLET_PORT"
         echo "Syncing the service..."
         sleep 10

run/mainnet/docker/.env

@@ -1 +1,3 @@
 NETWORK="mainnet"
+AGGREGATOR_ENDPOINT=https://aggregator.release-mainnet.api.mithril.network/aggregator
+GENESIS_VERIFICATION_KEY=5b3139312c36362c3134302c3138352c3133382c31312c3233372c3230372c3235302c3134342c32372c322c3138382c33302c31322c38312c3135352c3230342c31302c3137392c37352c32332c3133382c3139362c3231372c352c31342c32302c35372c37392c33392c3137365d

run/mainnet/docker/docker-compose.yml

@@ -45,3 +45,13 @@ services:
         compress: "true"
         max-file: "10"
         max-size: "50m"
+
+  mithril:
+    env_file:
+      - .env
+    image: ghcr.io/input-output-hk/mithril-client:test-csd-44dc188
+    user: ${USER_ID}:${GROUP_ID}
+    volumes:
+      - ${NODE_DB}:/app/db
+    profiles:
+      - mithril

run/mainnet/nix/.env

@@ -1 +1,3 @@
 NETWORK="mainnet"
+AGGREGATOR_ENDPOINT=https://aggregator.release-mainnet.api.mithril.network/aggregator
+GENESIS_VERIFICATION_KEY=5b3139312c36362c3134302c3138352c3133382c31312c3233372c3230372c3235302c3134342c32372c322c3138382c33302c31322c38312c3135352c3230342c31302c3137392c37352c32332c3133382c3139362c3231372c352c31342c32302c35372c37392c33392c3137365d