diff --git a/addons/mongodb/dataprotection/backup-info-collector.sh b/addons/mongodb/dataprotection/backup-info-collector.sh index 8cb75b0119..919ac5ac6b 100644 --- a/addons/mongodb/dataprotection/backup-info-collector.sh +++ b/addons/mongodb/dataprotection/backup-info-collector.sh @@ -1,25 +1,132 @@ # shellcheck disable=SC2148 function get_current_time() { - if [ -n "$(whereis mongosh | awk '{print $2}')" ]; then - CLIENT="mongosh" + local client curr_time status + + if command -v mongosh >/dev/null 2>&1; then + client="mongosh" + elif command -v mongo >/dev/null 2>&1; then + client="mongo" + else + printf 'backup info timestamp client unavailable rc=127\n' >&2 + return 127 + fi + + if curr_time=$( + "$client" \ + -u "$DP_DB_USER" \ + -p "$DP_DB_PASSWORD" \ + --port "$DP_DB_PORT" \ + --host "$DP_DB_HOST" \ + --authenticationDatabase admin \ + --eval 'db.isMaster().lastWrite.lastWriteDate.getTime()/1000' \ + --quiet + ); then + : else - CLIENT="mongo" + status=$? + printf 'backup info timestamp client failed rc=%s\n' "$status" >&2 + return "$status" fi - curr_time=$(${CLIENT} -u ${DP_DB_USER} -p ${DP_DB_PASSWORD} --port ${DP_DB_PORT} --host ${DP_DB_HOST} --authenticationDatabase admin --eval 'db.isMaster().lastWrite.lastWriteDate.getTime()/1000' --quiet) - curr_time=$(date -d "@${curr_time}" -u '+%Y-%m-%dT%H:%M:%SZ') - echo $curr_time + + if [ -z "$curr_time" ]; then + printf 'backup info timestamp client returned empty output rc=65\n' >&2 + return 65 + fi + + if curr_time=$(date -d "@${curr_time}" -u '+%Y-%m-%dT%H:%M:%SZ'); then + : + else + status=$? + printf 'backup info timestamp conversion failed rc=%s\n' "$status" >&2 + return "$status" + fi + + if [ -z "$curr_time" ]; then + printf 'backup info timestamp conversion returned empty output rc=65\n' >&2 + return 65 + fi + + printf '%s\n' "$curr_time" } function stat_and_save_backup_info() { + local start_time stop_time stat_output status total_size tmp_file + export PATH="$PATH:$DP_DATASAFED_BIN_PATH" export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" - START_TIME=$1 - STOP_TIME=$2 - if [ -z $STOP_TIME ]; then - STOP_TIME=$(get_current_time) + + start_time=${1:-} + stop_time=${2:-} + if [ -z "$stop_time" ]; then + if stop_time=$(get_current_time); then + : + else + status=$? + printf 'backup info stop timestamp failed rc=%s\n' "$status" >&2 + return "$status" + fi + fi + + if stat_output=$(datasafed stat /); then + : + else + status=$? + printf 'backup info datasafed stat failed rc=%s\n' "$status" >&2 + return "$status" + fi + + if total_size=$( + printf '%s\n' "$stat_output" | + awk ' + $1 == "TotalSize:" { + count++ + if (NF == 2 && $2 ~ /^[0-9]+$/) { + valid++ + value = $2 + } + } + END { + if (count == 1 && valid == 1) { + print value + exit 0 + } + exit 65 + } + ' + ); then + : + else + printf 'backup info TotalSize validation failed rc=65\n' >&2 + return 65 + fi + + if tmp_file=$(mktemp "${DP_BACKUP_INFO_FILE}.tmp.XXXXXX"); then + : + else + status=$? + printf 'backup info temporary file creation failed rc=%s\n' "$status" >&2 + return "$status" fi - TOTAL_SIZE=$(datasafed stat / | grep TotalSize | awk '{print $2}') - echo "{\"totalSize\":\"$TOTAL_SIZE\",\"timeRange\":{\"start\":\"${START_TIME}\",\"end\":\"${STOP_TIME}\"}}" >"${DP_BACKUP_INFO_FILE}" + + if printf '{"totalSize":"%s","timeRange":{"start":"%s","end":"%s"}}\n' \ + "$total_size" "$start_time" "$stop_time" >"$tmp_file"; then + : + else + status=$? + rm -f "$tmp_file" || : + printf 'backup info temporary file write failed rc=%s\n' "$status" >&2 + return "$status" + fi + + if mv "$tmp_file" "$DP_BACKUP_INFO_FILE"; then + return 0 + else + status=$? + fi + + rm -f "$tmp_file" || : + printf 'backup info publication failed rc=%s\n' "$status" >&2 + return "$status" } # if the script exits with a non-zero exit code, touch a file to indicate that the backup failed, diff --git a/addons/mongodb/dataprotection/datafile-backup.sh b/addons/mongodb/dataprotection/datafile-backup.sh index 146cc7026c..efb75e8182 100644 --- a/addons/mongodb/dataprotection/datafile-backup.sh +++ b/addons/mongodb/dataprotection/datafile-backup.sh @@ -1,9 +1,16 @@ +# shellcheck shell=bash set -o pipefail export PATH="$PATH:$DP_DATASAFED_BIN_PATH" export DATASAFED_BACKEND_BASE_PATH="$DP_BACKUP_BASE_PATH" trap handle_exit EXIT -cd ${DATA_DIR} -START_TIME=$(get_current_time) +# shellcheck disable=SC2164 +cd "$DATA_DIR" +if START_TIME=$(get_current_time); then + : +else + status=$? + exit "$status" +fi # TODO: flush data and locked write, otherwise data maybe inconsistent # NOTE: if files changed during taring, the exit code will be 1 when it ends. tar -cvf - ./ | datasafed push -z zstd-fastest - "${DP_BACKUP_NAME}.tar.zst" diff --git a/addons/mongodb/scripts-ut-spec/backup_info_collector_spec.sh b/addons/mongodb/scripts-ut-spec/backup_info_collector_spec.sh new file mode 100644 index 0000000000..22c0ad33ee --- /dev/null +++ b/addons/mongodb/scripts-ut-spec/backup_info_collector_spec.sh @@ -0,0 +1,377 @@ +# shellcheck shell=bash disable=SC1091,SC2016 + +Describe "MongoDB backup info collector contract" + setup_backup_info_collector() { + test_root=$(mktemp -d "${TMPDIR:-/tmp}/mongodb-backup-info-spec.XXXXXX") + fake_bin="$test_root/bin" + original_path=$PATH + real_mv=$(command -v mv) + mkdir -p "$fake_bin" + + cat >"$fake_bin/whereis" <<'SH' +#!/bin/sh +printf 'mongosh: %s\n' "$MONGODB_TEST_MONGOSH_BIN" +SH + + cat >"$fake_bin/mongosh" <<'SH' +#!/bin/sh +if [ -n "${MONGODB_TEST_CLIENT_OUTPUT:-}" ]; then + printf '%s\n' "$MONGODB_TEST_CLIENT_OUTPUT" +fi +exit "${MONGODB_TEST_CLIENT_RC:-0}" +SH + + cat >"$fake_bin/date" <<'SH' +#!/bin/sh +if [ -n "${MONGODB_TEST_DATE_OUTPUT:-}" ]; then + printf '%s\n' "$MONGODB_TEST_DATE_OUTPUT" +fi +exit "${MONGODB_TEST_DATE_RC:-0}" +SH + + cat >"$fake_bin/datasafed" <<'SH' +#!/bin/sh +case "${1:-}" in + stat) + if [ -n "${MONGODB_TEST_DATASAFED_OUTPUT:-}" ]; then + printf '%s\n' "$MONGODB_TEST_DATASAFED_OUTPUT" + fi + exit "${MONGODB_TEST_DATASAFED_RC:-0}" + ;; + push) + cat >/dev/null + exit 0 + ;; +esac +exit 64 +SH + + cat >"$fake_bin/tar" <<'SH' +#!/bin/sh +: >"$MONGODB_TEST_TAR_MARKER" +exit 0 +SH + + cat >"$fake_bin/mongodump" <<'SH' +#!/bin/sh +printf 'archive-data\n' +exit 0 +SH + + cat >"$fake_bin/mv" <<'SH' +#!/bin/sh +if [ -n "${MONGODB_TEST_MV_RC:-}" ]; then + exit "$MONGODB_TEST_MV_RC" +fi +exec "$MONGODB_TEST_REAL_MV" "$@" +SH + + chmod +x "$fake_bin/whereis" "$fake_bin/mongosh" \ + "$fake_bin/date" "$fake_bin/datasafed" "$fake_bin/tar" \ + "$fake_bin/mongodump" "$fake_bin/mv" + + export PATH="$fake_bin:$original_path" + export MONGODB_TEST_MONGOSH_BIN="$fake_bin/mongosh" + export MONGODB_TEST_REAL_MV="$real_mv" + export DP_DB_USER="backup-user" + export DP_DB_PASSWORD="backup-password" + export DP_DB_PORT="27017" + export DP_DB_HOST="mongodb.example" + export DP_DATASAFED_BIN_PATH="$fake_bin" + export DP_BACKUP_BASE_PATH="/backup/base" + export DP_BACKUP_INFO_FILE="$test_root/backup-info.json" + export DATA_DIR="$test_root/data" + export MONGODB_TEST_TAR_MARKER="$test_root/tar-called" + export MONGODB_TEST_CHILD_VERSION_FILE="$test_root/child-bash-version" + export DP_BACKUP_NAME="backup-name" + export PARALLEL="1" + mkdir -p "$DATA_DIR" + unset MONGODB_TEST_CLIENT_OUTPUT + unset MONGODB_TEST_CLIENT_RC + unset MONGODB_TEST_DATE_OUTPUT + unset MONGODB_TEST_DATE_RC + unset MONGODB_TEST_DATASAFED_OUTPUT + unset MONGODB_TEST_DATASAFED_RC + unset MONGODB_TEST_MV_RC + } + Before "setup_backup_info_collector" + + cleanup_backup_info_collector() { + PATH=$original_path + export PATH + rm -rf "$test_root" + unset test_root fake_bin original_path real_mv + unset MONGODB_TEST_MONGOSH_BIN + unset MONGODB_TEST_REAL_MV + unset MONGODB_TEST_CLIENT_OUTPUT + unset MONGODB_TEST_CLIENT_RC + unset MONGODB_TEST_DATE_OUTPUT + unset MONGODB_TEST_DATE_RC + unset MONGODB_TEST_DATASAFED_OUTPUT + unset MONGODB_TEST_DATASAFED_RC + unset MONGODB_TEST_MV_RC + unset DP_DB_USER DP_DB_PASSWORD DP_DB_PORT DP_DB_HOST + unset DP_DATASAFED_BIN_PATH DP_BACKUP_BASE_PATH DP_BACKUP_INFO_FILE + unset DATA_DIR MONGODB_TEST_TAR_MARKER MONGODB_TEST_CHILD_VERSION_FILE + unset DP_BACKUP_NAME PARALLEL + } + After "cleanup_backup_info_collector" + + run_get_current_time() { + source ../dataprotection/backup-info-collector.sh + get_current_time + } + + run_stat_and_report_file() { + local status + + source ../dataprotection/backup-info-collector.sh + stat_and_save_backup_info \ + "2026-07-29T00:00:00Z" "2026-07-29T00:05:00Z" + status=$? + if [ -f "$DP_BACKUP_INFO_FILE" ]; then + printf 'file=present\n' + cat "$DP_BACKUP_INFO_FILE" + else + printf 'file=absent\n' + fi + return "$status" + } + + run_datafile_backup_and_report() { + local child_version status tar_state info_state exit_marker residue_count + + "$SHELLSPEC_SHELL" -c ' + printf "%s\n" "$BASH_VERSION" >"$MONGODB_TEST_CHILD_VERSION_FILE" + source ../dataprotection/backup-info-collector.sh + source ../dataprotection/datafile-backup.sh + ' + status=$? + child_version=$(cat "$MONGODB_TEST_CHILD_VERSION_FILE") + if [ -f "$MONGODB_TEST_TAR_MARKER" ]; then + tar_state=present + else + tar_state=absent + fi + if [ -f "$DP_BACKUP_INFO_FILE" ]; then + info_state=present + else + info_state=absent + fi + if [ -f "${DP_BACKUP_INFO_FILE}.exit" ]; then + exit_marker=present + else + exit_marker=absent + fi + residue_count=$(find "$test_root" -maxdepth 1 -type f \ + ! -name backup-info.json ! -name tar-called \ + ! -name backup-info.json.exit ! -name child-bash-version \ + | wc -l | tr -d ' ') + printf 'tar=%s info=%s marker=%s residue=%s child=%s\n' \ + "$tar_state" "$info_state" "$exit_marker" "$residue_count" \ + "$child_version" + return "$status" + } + + run_empty_time_contract() { + local helper_status action_status + + source ../dataprotection/backup-info-collector.sh + get_current_time >/dev/null + helper_status=$? + run_datafile_backup_and_report + action_status=$? + printf 'helper=%s action=%s\n' "$helper_status" "$action_status" + return "$action_status" + } + + run_stat_with_existing_destination() { + local destination residue_count status + + printf 'previous-metadata\n' >"$DP_BACKUP_INFO_FILE" + source ../dataprotection/backup-info-collector.sh + stat_and_save_backup_info \ + "2026-07-29T00:00:00Z" "2026-07-29T00:05:00Z" + status=$? + destination=$(cat "$DP_BACKUP_INFO_FILE") + residue_count=$(find "$test_root" -maxdepth 1 -type f \ + ! -name backup-info.json ! -name tar-called | wc -l | tr -d ' ') + printf 'destination=%s residue=%s\n' "$destination" "$residue_count" + return "$status" + } + + run_mongodump_publication_failure() { + local child_version destination exit_marker residue_count status + + printf 'previous-metadata\n' >"$DP_BACKUP_INFO_FILE" + "$SHELLSPEC_SHELL" -c ' + printf "%s\n" "$BASH_VERSION" >"$MONGODB_TEST_CHILD_VERSION_FILE" + source ../dataprotection/backup-info-collector.sh + source ../dataprotection/mongodump-backup.sh + ' + status=$? + child_version=$(cat "$MONGODB_TEST_CHILD_VERSION_FILE") + destination=$(cat "$DP_BACKUP_INFO_FILE") + if [ -f "${DP_BACKUP_INFO_FILE}.exit" ]; then + exit_marker=present + else + exit_marker=absent + fi + residue_count=$(find "$test_root" -maxdepth 1 -type f \ + ! -name backup-info.json ! -name backup-info.json.exit \ + ! -name tar-called ! -name child-bash-version | wc -l | tr -d ' ') + printf 'destination=%s marker=%s residue=%s child=%s\n' \ + "$destination" "$exit_marker" "$residue_count" "$child_version" + return "$status" + } + + It "preserves the MongoDB client failure instead of fabricating a timestamp" + export MONGODB_TEST_CLIENT_RC=17 + export MONGODB_TEST_DATE_OUTPUT="2026-07-29T00:00:00Z" + + When call run_get_current_time + + The status should equal 17 + The output should be blank + The stderr should equal "backup info timestamp client failed rc=17" + End + + It "fails when the client timestamp cannot be converted" + export MONGODB_TEST_CLIENT_OUTPUT="not-an-epoch" + export MONGODB_TEST_DATE_RC=11 + + When call run_get_current_time + + The status should equal 11 + The output should be blank + The stderr should equal "backup info timestamp conversion failed rc=11" + End + + It "classifies empty successful MongoDB output before the datafile backup" + export MONGODB_TEST_DATE_OUTPUT="2026-07-29T00:00:00Z" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize: 42" + + When call run_empty_time_contract + + The status should equal 1 + The line 1 of output should equal "failed with exit code 65" + The line 2 of output should equal "tar=absent info=absent marker=present residue=0 child=$BASH_VERSION" + The line 3 of output should equal "helper=65 action=1" + The stderr should equal "backup info timestamp client returned empty output rc=65 +backup info timestamp client returned empty output rc=65" + End + + It "classifies empty successful date output before the datafile backup" + export MONGODB_TEST_CLIENT_OUTPUT="1753747200" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize: 42" + + When call run_empty_time_contract + + The status should equal 1 + The line 1 of output should equal "failed with exit code 65" + The line 2 of output should equal "tar=absent info=absent marker=present residue=0 child=$BASH_VERSION" + The line 3 of output should equal "helper=65 action=1" + The stderr should equal "backup info timestamp conversion returned empty output rc=65 +backup info timestamp conversion returned empty output rc=65" + End + + It "preserves datasafed failure and does not write backup metadata" + export MONGODB_TEST_DATASAFED_RC=23 + + When call run_stat_and_report_file + + The status should equal 23 + The output should equal "file=absent" + The stderr should equal "backup info datasafed stat failed rc=23" + End + + It "rejects a successful datasafed response without TotalSize" + export MONGODB_TEST_DATASAFED_OUTPUT="Files 4" + + When call run_stat_and_report_file + + The status should be failure + The output should equal "file=absent" + The stderr should equal "backup info TotalSize validation failed rc=65" + End + + It "rejects an empty TotalSize value without replacing existing metadata" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize:" + + When call run_stat_with_existing_destination + + The status should equal 65 + The output should equal "destination=previous-metadata residue=0" + The stderr should equal "backup info TotalSize validation failed rc=65" + End + + It "rejects duplicate TotalSize values without replacing existing metadata" + MONGODB_TEST_DATASAFED_OUTPUT='TotalSize: 42 +TotalSize: 84' + export MONGODB_TEST_DATASAFED_OUTPUT + + When call run_stat_with_existing_destination + + The status should equal 65 + The output should equal "destination=previous-metadata residue=0" + The stderr should equal "backup info TotalSize validation failed rc=65" + End + + It "rejects a JSON-unsafe TotalSize value without replacing existing metadata" + export MONGODB_TEST_DATASAFED_OUTPUT='TotalSize: "' + + When call run_stat_with_existing_destination + + The status should equal 65 + The output should equal "destination=previous-metadata residue=0" + The stderr should equal "backup info TotalSize validation failed rc=65" + End + + It "writes exact metadata for a valid datasafed TotalSize response" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize: 42" + + When call run_stat_and_report_file + + The status should be success + The line 1 of output should equal "file=present" + The line 2 of output should equal '{"totalSize":"42","timeRange":{"start":"2026-07-29T00:00:00Z","end":"2026-07-29T00:05:00Z"}}' + End + + It "stops the datafile backup before tar when the start timestamp fails" + export MONGODB_TEST_CLIENT_RC=17 + export MONGODB_TEST_DATE_OUTPUT="2026-07-29T00:00:00Z" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize: 42" + + When call run_datafile_backup_and_report + + The status should equal 1 + The line 1 of output should equal "failed with exit code 17" + The line 2 of output should equal "tar=absent info=absent marker=present residue=0 child=$BASH_VERSION" + The stderr should equal "backup info timestamp client failed rc=17" + End + + It "preserves existing metadata and removes temporary output when publication fails" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize: 42" + export MONGODB_TEST_MV_RC=29 + + When call run_stat_with_existing_destination + + The status should equal 29 + The output should equal "destination=previous-metadata residue=0" + The stderr should equal "backup info publication failed rc=29" + End + + It "cleans failed publication under mongodump errexit and records action failure" + export MONGODB_TEST_CLIENT_OUTPUT="1753747200" + export MONGODB_TEST_DATE_OUTPUT="2026-07-29T00:00:00Z" + export MONGODB_TEST_DATASAFED_OUTPUT="TotalSize: 42" + export MONGODB_TEST_MV_RC=29 + + When call run_mongodump_publication_failure + + The status should equal 1 + The line 1 of output should equal "failed with exit code 29" + The line 2 of output should equal "destination=previous-metadata marker=present residue=0 child=$BASH_VERSION" + The stderr should equal "backup info publication failed rc=29" + End +End