diff --git a/addons/mariadb/scripts-ut-spec/mariadb_image_entrypoint_retry_spec.sh b/addons/mariadb/scripts-ut-spec/mariadb_image_entrypoint_retry_spec.sh new file mode 100644 index 0000000000..ba50b16e70 --- /dev/null +++ b/addons/mariadb/scripts-ut-spec/mariadb_image_entrypoint_retry_spec.sh @@ -0,0 +1,207 @@ +# shellcheck shell=sh + +Describe "MariaDB image entrypoint fresh-init retry contract" + Include ../scripts/mariadb-image-entrypoint.sh + + setup() { + TEST_ROOT="$(mktemp -d)" + export TEST_ROOT + } + + cleanup() { + rm -rf "${TEST_ROOT}" + } + + BeforeEach 'setup' + AfterEach 'cleanup' + + make_late_ready_fixture() { + fixture="$1" + cat > "${fixture}" <<'FIXTURE' +#!/bin/bash +set -e +DATABASE_ALREADY_EXISTS= +DATADIR="${TEST_DATADIR:?}" +attempt_file="${TEST_ATTEMPT_FILE:?}" + +docker_process_sql() { + local attempts=0 + [ ! -f "${attempt_file}" ] || attempts="$(cat "${attempt_file}")" + attempts=$((attempts + 1)) + printf '%s\n' "${attempts}" > "${attempt_file}" + [ "${attempts}" -ge "${TEST_READY_AT:-33}" ] +} +sleep() { :; } +mysql_error() { + printf '%s\n' "$*" >&2 + exit 1 +} +docker_temp_server_start() { + extraArgs=() + local i + for i in {30..0}; do + if docker_process_sql "${extraArgs[@]}" --database=mysql <<<'SELECT 1' &>/dev/null; then + break + fi + sleep 1 + done + if [ "$i" = 0 ]; then + mysql_error "Unable to start server." + fi +} +docker_mariadb_init() { + docker_temp_server_start + touch "${DATADIR}/account-setup-committed" +} +_main() { + docker_mariadb_init "$@" +} +_main "$@" +FIXTURE + chmod 0555 "${fixture}" + } + + run_unpatched_late_ready_fixture() { + fixture="${TEST_ROOT}/docker-entrypoint.sh" + make_late_ready_fixture "${fixture}" + mkdir -p "${TEST_ROOT}/data" + TEST_DATADIR="${TEST_ROOT}/data" \ + TEST_ATTEMPT_FILE="${TEST_ROOT}/attempts" \ + bash "${fixture}" + } + + run_patched_late_ready_fixture() { + fixture="${TEST_ROOT}/docker-entrypoint.sh" + patched="${TEST_ROOT}/docker-entrypoint.patched.sh" + make_late_ready_fixture "${fixture}" + mkdir -p "${TEST_ROOT}/data" + touch "${TEST_ROOT}/data/.kb-mariadb-image-init-in-progress" + prepare_mariadb_image_entrypoint "${fixture}" "${patched}" 120 + TEST_DATADIR="${TEST_ROOT}/data" \ + TEST_ATTEMPT_FILE="${TEST_ROOT}/attempts" \ + bash "${patched}" + attempts="$(cat "${TEST_ROOT}/attempts")" + committed=false + complete=false + in_progress=true + [ -f "${TEST_ROOT}/data/account-setup-committed" ] && committed=true + [ -f "${TEST_ROOT}/data/.kb-mariadb-image-init-complete" ] && complete=true + [ ! -f "${TEST_ROOT}/data/.kb-mariadb-image-init-in-progress" ] && in_progress=false + printf 'attempts=%s committed=%s complete=%s in_progress=%s\n' \ + "${attempts}" "${committed}" "${complete}" "${in_progress}" + } + + run_patched_after_budget_fixture() { + fixture="${TEST_ROOT}/docker-entrypoint.sh" + patched="${TEST_ROOT}/docker-entrypoint.patched.sh" + make_late_ready_fixture "${fixture}" + mkdir -p "${TEST_ROOT}/data" + touch "${TEST_ROOT}/data/.kb-mariadb-image-init-in-progress" + prepare_mariadb_image_entrypoint "${fixture}" "${patched}" 120 + rc=0 + TEST_READY_AT=121 \ + TEST_DATADIR="${TEST_ROOT}/data" \ + TEST_ATTEMPT_FILE="${TEST_ROOT}/attempts" \ + bash "${patched}" 2>"${TEST_ROOT}/stderr" || rc=$? + attempts="$(cat "${TEST_ROOT}/attempts")" + committed=false + complete=false + in_progress=false + [ -f "${TEST_ROOT}/data/account-setup-committed" ] && committed=true + [ -f "${TEST_ROOT}/data/.kb-mariadb-image-init-complete" ] && complete=true + [ -f "${TEST_ROOT}/data/.kb-mariadb-image-init-in-progress" ] && in_progress=true + error="$(tr '\n' ' ' < "${TEST_ROOT}/stderr")" + printf 'rc=%s attempts=%s committed=%s complete=%s in_progress=%s error=%s\n' \ + "${rc}" "${attempts}" "${committed}" "${complete}" "${in_progress}" "${error}" + } + + recover_partial_init() { + data="${TEST_ROOT}/data" + mkdir -p "${data}/mysql" "${data}/runtime-overrides.d" "${data}/log" "${data}/binlog" "${data}/tmp" + touch "${data}/mysql/user.frm" "${data}/ibdata1" + printf 'keep\n' > "${data}/runtime-overrides.cnf" + printf 'evidence\n' > "${data}/log/entrypoint.log" + touch "${data}/.kb-mariadb-image-init-in-progress" + recover_partial_mariadb_image_init "${data}" + mysql_absent=false + ibdata_absent=false + runtime_kept=false + evidence_kept=false + marker_absent=false + [ ! -e "${data}/mysql" ] && mysql_absent=true + [ ! -e "${data}/ibdata1" ] && ibdata_absent=true + [ -f "${data}/runtime-overrides.cnf" ] && runtime_kept=true + [ -f "${data}/log/entrypoint.log" ] && evidence_kept=true + [ ! -e "${data}/.kb-mariadb-image-init-in-progress" ] && marker_absent=true + printf 'mysql_absent=%s ibdata_absent=%s runtime_kept=%s evidence_kept=%s marker_absent=%s\n' \ + "${mysql_absent}" "${ibdata_absent}" "${runtime_kept}" "${evidence_kept}" "${marker_absent}" + } + + preserve_legacy_existing_data() { + data="${TEST_ROOT}/data" + mkdir -p "${data}/mysql" + printf 'legacy\n' > "${data}/mysql/user.frm" + recover_partial_mariadb_image_init "${data}" + cat "${data}/mysql/user.frm" + } + + wrapper_is_mounted_and_called_before_existing_data_detection() { + configmap="${SHELLSPEC_CWD:?}/addons/mariadb/templates/configmap-scripts-replication.yaml" + entrypoint="${SHELLSPEC_CWD:?}/addons/mariadb/scripts/replication-entrypoint.sh" + mount_count="$(grep -c 'mariadb-image-entrypoint.sh:' "${configmap}")" + recover_line="$(grep -n '/scripts/mariadb-image-entrypoint.sh recover "${DATA_DIR}"' "${entrypoint}" | cut -d: -f1)" + existing_line="$(grep -n '^HAS_EXISTING_DATA=false$' "${entrypoint}" | cut -d: -f1)" + run_count="$(grep -c '/scripts/mariadb-image-entrypoint.sh run "${DATA_DIR}" mariadbd' "${entrypoint}")" + printf 'mount=%s recover_before_existing=%s run=%s\n' \ + "${mount_count}" "$([ "${recover_line}" -lt "${existing_line}" ] && echo true || echo false)" "${run_count}" + } + + It "reproduces the upstream 30-attempt boundary before the addon patch" + When call run_unpatched_late_ready_fixture + The status should be failure + The stderr should include "Unable to start server." + End + + It "admits a server that becomes ready on attempt 33 and commits account setup" + When call run_patched_late_ready_fixture + The status should be success + The output should equal "attempts=33 committed=true complete=true in_progress=false" + End + + It "fails closed after the 120-second budget without committing partial initialization" + When call run_patched_after_budget_fixture + The status should be success + The output should equal "rc=1 attempts=121 committed=false complete=false in_progress=true error=Unable to start server. " + End + + It "recovers only addon-owned interrupted fresh initialization while preserving config and evidence" + When call recover_partial_init + The status should be success + The output should equal "mysql_absent=true ibdata_absent=true runtime_kept=true evidence_kept=true marker_absent=true" + The stderr should include "recovering interrupted addon-owned MariaDB fresh initialization" + End + + It "does not erase legacy existing data without the addon in-progress marker" + When call preserve_legacy_existing_data + The status should be success + The output should equal "legacy" + End + + It "mounts the wrapper, recovers before existing-data classification, and owns the image-entrypoint launch" + When call wrapper_is_mounted_and_called_before_existing_data_detection + The status should be success + The output should equal "mount=1 recover_before_existing=true run=1" + End + + It "rejects unsafe datadir recovery targets" + When call recover_partial_mariadb_image_init "/" + The status should be failure + The stderr should include "unsafe MariaDB data directory" + End + + It "rejects an unbounded temporary-server startup timeout" + When call validate_temp_server_start_timeout "301" + The status should be failure + The stderr should include "between 31 and 300 seconds" + End +End diff --git a/addons/mariadb/scripts-ut-spec/reconfigure_persisted_alpha86_spec.sh b/addons/mariadb/scripts-ut-spec/reconfigure_persisted_alpha86_spec.sh index 512d5e4263..fe7650ca32 100644 --- a/addons/mariadb/scripts-ut-spec/reconfigure_persisted_alpha86_spec.sh +++ b/addons/mariadb/scripts-ut-spec/reconfigure_persisted_alpha86_spec.sh @@ -196,12 +196,12 @@ Describe "alpha.86 reconfigureAction.persisted semisync gates" It "the --defaults-extra-file flag appears immediately after the mariadbd command (positional first arg)" # MariaDB requires --defaults-* flags to be the first option. # We assert by source-order: the line containing - # `docker-entrypoint.sh mariadbd \` must be IMMEDIATELY followed + # The addon-owned image-entrypoint wrapper invocation must be IMMEDIATELY followed # by the `--defaults-extra-file=...` line (allowing only # whitespace and line-continuation markers). When run sh -c ' awk " - /docker-entrypoint\.sh mariadbd \\\\/ { found_mariadbd = NR; next } + /mariadb-image-entrypoint\.sh run .* mariadbd \\\\/ { found_mariadbd = NR; next } found_mariadbd && NR == found_mariadbd + 1 { if (\$0 ~ /--defaults-extra-file=/) { print \"OK\" diff --git a/addons/mariadb/scripts-ut-spec/replication_entrypoint_readonly_spec.sh b/addons/mariadb/scripts-ut-spec/replication_entrypoint_readonly_spec.sh index 7d29174111..31a4082991 100644 --- a/addons/mariadb/scripts-ut-spec/replication_entrypoint_readonly_spec.sh +++ b/addons/mariadb/scripts-ut-spec/replication_entrypoint_readonly_spec.sh @@ -14,10 +14,10 @@ Describe "replication-entrypoint read_only fail-closed startup contract" ENTRYPOINT="${SHELLSPEC_CWD:?}/addons/mariadb/scripts/replication-entrypoint.sh" extract_mariadbd_launch() { - # The bootstrap launch: from the `docker-entrypoint.sh mariadbd \` + # The bootstrap launch: from the addon-owned image-entrypoint wrapper # line to the line ending the backslash-continued argv (bind-address). awk ' - /docker-entrypoint\.sh mariadbd \\/ { in_cmd = 1 } + /mariadb-image-entrypoint\.sh run .* mariadbd \\/ { in_cmd = 1 } in_cmd { print } in_cmd && /--bind-address=/ { exit } ' "${ENTRYPOINT}" diff --git a/addons/mariadb/scripts-ut-spec/replication_full_accept_user_fence_spec.sh b/addons/mariadb/scripts-ut-spec/replication_full_accept_user_fence_spec.sh index 1604cdb719..242d5027ec 100644 --- a/addons/mariadb/scripts-ut-spec/replication_full_accept_user_fence_spec.sh +++ b/addons/mariadb/scripts-ut-spec/replication_full_accept_user_fence_spec.sh @@ -38,6 +38,11 @@ Describe "replication full-primary acceptance user fence" extract_function read_only_is_strongest_fail_closed extract_function rollback_fenced_primary_accept extract_function rollback_locked_primary_accept + extract_function force_release_primary_write_publication_lock_after_commit_receipt + extract_function primary_write_publication_receipt_matches + extract_function recover_committed_primary_write_publication + extract_function rollback_ambiguous_primary_publish + extract_function persist_primary_write_accept_guard extract_function set_primary_read_write cat <<'HARNESS' if [ "${MODE}" = "entry-not-fenced" ]; then @@ -52,10 +57,25 @@ REMOTE_ROOT_LOCKED=1 REQUIRED_GATES_PASSED=0 POST_COMMIT_RETURNED=0 DEMOTED_AFTER_COMMIT=0 +DRIFTED_BEFORE_FIRST_MARKER=0 +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}" } +touch() { + if [ "${MODE}" = "authority-drift-after-check-before-first-marker" ] && \ + [ "$1" = "${DATA_DIR}/.primary-read-write-ready" ] && \ + [ "${DRIFTED_BEFORE_FIRST_MARKER}" -eq 0 ]; then + GLOBAL_READ_ONLY=1 + READ_ONLY_MODE=ON + DRIFTED_BEFORE_FIRST_MARKER=1 + command rm -f "${DATA_DIR}/.primary-read-write-ready" "${DATA_DIR}/.replication-ready" + trace_event authority-drift-after-check-before-first-marker + fi + command touch "$@" +} ordinary_business_can_write() { [ "${GLOBAL_READ_ONLY}" -eq 0 ] } @@ -113,6 +133,16 @@ primary_internal_root_write_ready() { trace_event required-gate-pass } query_local_syncer_role() { + if [ "${MODE}" = "authority-drift-after-check-before-first-marker" ] && \ + [ "${DRIFTED_BEFORE_FIRST_MARKER}" -eq 1 ]; then + printf '%s\n' secondary + return 0 + fi + if [ "${MODE}" = "authority-drift-before-first-marker" ] && [ "${POST_COMMIT_RETURNED}" -eq 1 ]; then + trace_event authority-drift-before-first-marker + printf '%s\n' secondary + return 0 + fi if [ "${MODE}" = "role-drift" ] || [ "${MODE}" = "role-drift-rollback-failure" ] || [ "${MODE}" = "role-drift-strongest-failure" ]; then printf '%s\n' secondary else @@ -128,6 +158,18 @@ release_primary_write_commit_lock() { trace_event commit-lock-released return 0 } +acquire_primary_write_publication_lock_for_rollback() { + trace_event publication-lock-acquired-by-caller-rollback + return 0 +} +release_primary_write_publication_lock() { + trace_event publication-lock-released-by-caller-rollback + return 0 +} +force_release_primary_write_commit_lock() { + trace_event commit-lock-force-released + return 0 +} inject_demote_after_commit_return() { if [ "${MODE}" = "post-commit-demote" ] && [ "${POST_COMMIT_RETURNED}" -eq 1 ] && [ "${DEMOTED_AFTER_COMMIT}" -eq 0 ]; then GLOBAL_READ_ONLY=1 @@ -155,16 +197,28 @@ authoritative_primary_write_commit() { GLOBAL_READ_ONLY=0 READ_ONLY_MODE=OFF trace_event global-read-only-off - if [ "${MODE}" = "ready-publish-failure" ]; then - trace_event ready-publish-failed-after-global-open - return 1 - fi - command touch "${DATA_DIR}/.primary-read-write-ready" "${DATA_DIR}/.replication-ready" - trace_event ready-published - trace_event replication-ready-published trace_event syncer-authority-commit-pass POST_COMMIT_RETURNED=1 } +authoritative_primary_write_publish() { + expected_token="$2" + if [ "${MODE}" = "publish-receipt-lost-after-guard-clear" ]; then + command rm -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" + trace_event syncer-cleared-guard-before-receipt-loss + return 1 + fi + if [ "${MODE}" = "authority-drift-before-first-marker" ]; then + trace_event authority-drift-before-publication-commit + return 1 + fi + if [ "${MODE}" = "authority-drift-after-check-before-first-marker" ]; then + trace_event authority-drift-after-marker-stage + return 1 + fi + printf '%s\n' "${expected_token}" > "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" + command rm -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" + trace_event syncer-publication-commit-pass +} local_sql() { case "$*" in *"SHOW GRANTS FOR"*) @@ -240,10 +294,19 @@ set_fail_closed_read_only() { fi } mark_replication_pending() { - rm -f "${DATA_DIR}/.primary-read-write-ready" + rm -f "${DATA_DIR}/.primary-read-write-ready" "${DATA_DIR}/.replication-ready" trace_event replication-pending } -mark_replication_ready() { trace_event unexpected-addon-ready-publication; return 1; } +mark_replication_ready() { + if [ "${MODE}" = "ready-publish-failure" ]; then + trace_event ready-publish-failed-after-global-open + return 1 + fi + command touch "${DATA_DIR}/.replication-ready" + command rm -f "${DATA_DIR}/.replication-pending" + trace_event ready-published + trace_event replication-ready-published +} prestop_watchdog_log() { trace_event "log:$*" } @@ -286,13 +349,13 @@ case "${MODE}" in [ "${remote_line}" -lt "${replication_ready_line}" ] [ "${replication_ready_line}" -lt "${commit_unlock_line}" ] grep -q '^ready-observed-after-return$' "${TRACE_FILE}" - ! grep -q '^unexpected-addon-ready-publication$' "${TRACE_FILE}" ! grep -q 'open-before-required-gates' "${TRACE_FILE}" grep -q '^ordinary-business-writable$' "${TRACE_FILE}" grep -q '^local-root-writable$' "${TRACE_FILE}" grep -q '^remote-root-writable$' "${TRACE_FILE}" + [ ! -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] ;; - prestop|gate-prestop|entry-not-fenced|bypass-failure|bypass-super|bypass-all|gate-mode-failure|gate-failure|authority-lost|open-failure|local-unlock-failure|remote-unlock-failure|ready-publish-failure|role-drift) + prestop|gate-prestop|entry-not-fenced|bypass-failure|bypass-super|bypass-all|gate-mode-failure|gate-failure|authority-lost|open-failure|local-unlock-failure|remote-unlock-failure|ready-publish-failure|role-drift|authority-drift-before-first-marker|authority-drift-after-check-before-first-marker|publish-receipt-lost-after-guard-clear) [ "${accept_rc}" -eq 2 ] grep -q '^ordinary-business-rejected$' "${TRACE_FILE}" grep -q '^local-root-rejected$' "${TRACE_FILE}" @@ -300,7 +363,27 @@ case "${MODE}" in [ "${READ_ONLY_MODE}" = "NO_LOCK_NO_ADMIN" ] [ "${LOCAL_ROOT_LOCKED}" -eq 1 ] [ "${REMOTE_ROOT_LOCKED}" -eq 1 ] - ! grep -q '^ready-published$' "${TRACE_FILE}" + if [ "${MODE}" = "authority-drift-before-first-marker" ] || \ + [ "${MODE}" = "authority-drift-after-check-before-first-marker" ] || \ + [ "${MODE}" = "publish-receipt-lost-after-guard-clear" ]; then + grep -q '^ready-published$' "${TRACE_FILE}" + [ ! -f "${DATA_DIR}/.primary-read-write-ready" ] + [ ! -f "${DATA_DIR}/.replication-ready" ] + else + ! grep -q '^ready-published$' "${TRACE_FILE}" + fi + if [ "${MODE}" = "authority-drift-before-first-marker" ] || \ + [ "${MODE}" = "authority-drift-after-check-before-first-marker" ] || \ + [ "${MODE}" = "publish-receipt-lost-after-guard-clear" ]; then + [ -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] + fi + if [ "${MODE}" = "publish-receipt-lost-after-guard-clear" ]; then + handler_line="$(grep -n '^syncer-cleared-guard-before-receipt-loss$' "${TRACE_FILE}" | cut -d: -f1)" + recovery_lock_line="$(grep -n '^publication-lock-acquired-by-caller-rollback$' "${TRACE_FILE}" | cut -d: -f1)" + recovery_unlock_line="$(grep -n '^publication-lock-released-by-caller-rollback$' "${TRACE_FILE}" | cut -d: -f1)" + [ "${handler_line}" -lt "${recovery_lock_line}" ] + [ "${recovery_lock_line}" -lt "${recovery_unlock_line}" ] + fi ;; rollback-failure|role-drift-rollback-failure|role-drift-strongest-failure) [ "${accept_rc}" -eq 3 ] @@ -311,13 +394,17 @@ case "${MODE}" in [ "${accept_rc}" -eq 0 ] [ "${READ_ONLY_MODE}" = "ON" ] grep -q '^run-cycle-demote-after-authority-commit$' "${TRACE_FILE}" + ready_line="$(grep -n '^replication-ready-published$' "${TRACE_FILE}" | cut -d: -f1)" + demote_line="$(grep -n '^run-cycle-demote-after-authority-commit$' "${TRACE_FILE}" | cut -d: -f1)" + unlock_line="$(grep -n '^commit-lock-released$' "${TRACE_FILE}" | cut -d: -f1)" + [ "${ready_line}" -lt "${demote_line}" ] + [ "${demote_line}" -lt "${unlock_line}" ] grep -q '^ordinary-business-rejected$' "${TRACE_FILE}" grep -q '^local-root-rejected$' "${TRACE_FILE}" grep -q '^remote-root-rejected$' "${TRACE_FILE}" [ ! -f "${DATA_DIR}/.primary-read-write-ready" ] [ ! -f "${DATA_DIR}/.replication-ready" ] ! grep -q '^visible-transition-after-demote$' "${TRACE_FILE}" - ! grep -q '^unexpected-addon-ready-publication$' "${TRACE_FILE}" ;; esac HARNESS @@ -329,6 +416,148 @@ HARNESS return "${rc}" } + run_mark_replication_ready_first_touch_failure() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}" + command touch "${data_dir}/.replication-pending" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function mark_replication_ready + cat <<'HARNESS' +touch() { + if [ "$1" = "${DATA_DIR}/.replication-ready" ]; then + return 1 + fi + command touch "$@" +} +rc=0 +mark_replication_ready || rc=$? +[ "${rc}" -ne 0 ] +[ -f "${DATA_DIR}/.replication-pending" ] +[ ! -f "${DATA_DIR}/.replication-ready" ] +HARNESS + } > "${harness}" + + DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_publication_handler_crash_after_guard_commit_recovers_exact_terminal() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + trace="${work_dir}/trace" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}/.primary-write-publication-lock" + printf '%s\n' reviewer-crash-boundary-transaction > \ + "${data_dir}/.primary-write-publication-committed" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function force_release_primary_write_publication_lock_after_commit_receipt + extract_function primary_write_publication_receipt_matches + extract_function recover_committed_primary_write_publication + extract_function rollback_ambiguous_primary_publish + cat <<'HARNESS' +PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_PUBLICATION_LOCK_DIR="${DATA_DIR}/.primary-write-publication-lock" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } +prestop_watchdog_log() { trace_event "log:$*"; } +acquire_primary_write_publication_lock_for_rollback() { + trace_event stale-publication-lock-timeout + return 1 +} +rollback_locked_primary_accept() { + trace_event unexpected-strongest-rollback + return 3 +} +release_primary_write_commit_lock() { + trace_event commit-lock-released + return 0 +} + +rc=0 +rollback_ambiguous_primary_publish reviewer-crash publication-commit-rejected \ + reviewer-crash-boundary-transaction || rc=$? +failure=0 +[ "${rc}" -eq 0 ] || failure=1 +[ ! -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] || failure=1 +[ -f "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" ] || failure=1 +[ ! -e "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" ] || failure=1 +grep -q 'publication=durable-exact-receipt recovery=ambiguous-response' "${TRACE_FILE}" || failure=1 +! grep -q '^unexpected-strongest-rollback$' "${TRACE_FILE}" || failure=1 +cat "${TRACE_FILE}" +exit "${failure}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_publication_handler_crash_without_exact_receipt_restores_guard() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + trace="${work_dir}/trace" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}/.primary-write-publication-lock" + printf '%s\n' different-transaction > \ + "${data_dir}/.primary-write-publication-committed" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function force_release_primary_write_publication_lock_after_commit_receipt + extract_function primary_write_publication_receipt_matches + extract_function recover_committed_primary_write_publication + extract_function persist_primary_write_accept_guard + extract_function rollback_ambiguous_primary_publish + cat <<'HARNESS' +PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_PUBLICATION_LOCK_DIR="${DATA_DIR}/.primary-write-publication-lock" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } +prestop_watchdog_log() { trace_event "log:$*"; } +acquire_primary_write_publication_lock_for_rollback() { + trace_event stale-publication-lock-timeout + return 1 +} +rollback_locked_primary_accept() { + trace_event strongest-rollback + return 2 +} + +expected_token=reviewer-crash-boundary-transaction +rc=0 +rollback_ambiguous_primary_publish reviewer-crash publication-commit-rejected \ + "${expected_token}" || rc=$? +failure=0 +[ "${rc}" -eq 3 ] || failure=1 +[ -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] || failure=1 +[ "$(cat "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" 2>/dev/null)" = "${expected_token}" ] || failure=1 +[ -e "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" ] || failure=1 +[ "$(cat "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}")" = different-transaction ] || failure=1 +grep -q '^strongest-rollback$' "${TRACE_FILE}" || failure=1 +! grep -q 'publication=durable-exact-receipt' "${TRACE_FILE}" || failure=1 +cat "${TRACE_FILE}" +exit "${failure}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + It "keeps ordinary, local-root, and remote-root writers fenced until all required gates pass" When call run_accept_case success The status should be success @@ -337,7 +566,7 @@ HARNESS The output should not include "open-before-required-gates" End - It "has no addon-visible transition after the authority commit returns" + It "does not republish readiness after a demote wins before commit-lock release" When call run_accept_case post-commit-demote The status should be success The output should include "run-cycle-demote-after-authority-commit" @@ -461,6 +690,50 @@ HARNESS The output should not include "ready-published" End + It "rejects authority drift after the writer-open receipt at the mutex-linearized publication commit" + When call run_accept_case authority-drift-before-first-marker + The status should be success + The output should include "syncer-authority-commit-pass" + The output should include "authority-drift-before-publication-commit" + The output should include "global-read-only-strongest" + The output should include "ready-published" + End + + It "rolls back when authority drifts after marker staging but before publication commit" + When call run_accept_case authority-drift-after-check-before-first-marker + The status should be success + The output should include "authority-drift-after-check-before-first-marker" + The output should include "authority-drift-after-marker-stage" + The output should include "global-read-only-strongest" + The output should include "ready-published" + The output should not include "ready-observed-after-return" + End + + It "recreates the guard before strongest rollback when the publication receipt is lost after server-side clear" + When call run_accept_case publish-receipt-lost-after-guard-clear + The status should be success + The output should include "syncer-cleared-guard-before-receipt-loss" + The output should include "global-read-only-strongest" + The output should include "local-root-locked" + The output should include "remote-root-locked" + The output should not include "ready-observed-after-return" + End + + It "recovers the exact durable terminal after the handler dies between guard commit and lock cleanup" + When call run_publication_handler_crash_after_guard_commit_recovers_exact_terminal + The status should be success + The output should include "publication=durable-exact-receipt recovery=ambiguous-response" + The output should not include "unexpected-strongest-rollback" + End + + It "restores the exact guard and stays fail-closed when a stale lock has no matching terminal receipt" + When call run_publication_handler_crash_without_exact_receipt_restores_guard + The status should be success + The output should include "stale-publication-lock-timeout" + The output should include "strongest-rollback" + The output should not include "publication=durable-exact-receipt" + End + It "rolls all user writers back when local-root unlock fails before the authority commit" When call run_accept_case local-unlock-failure The status should be success @@ -477,6 +750,11 @@ HARNESS The output should not include "ready-published" End + It "returns failure immediately when the first replication-ready marker touch fails" + When call run_mark_replication_ready_first_touch_failure + The status should be success + End + It "surfaces rollback failure instead of publishing a false fail-closed state" When call run_accept_case rollback-failure The status should be success diff --git a/addons/mariadb/scripts-ut-spec/replication_merged_semisync_startup_recovery_spec.sh b/addons/mariadb/scripts-ut-spec/replication_merged_semisync_startup_recovery_spec.sh index d0e6953d56..76a345dbc4 100644 --- a/addons/mariadb/scripts-ut-spec/replication_merged_semisync_startup_recovery_spec.sh +++ b/addons/mariadb/scripts-ut-spec/replication_merged_semisync_startup_recovery_spec.sh @@ -212,7 +212,7 @@ Describe "cmpd-replication.yaml semisync startup recovery" End It "keeps the preStop fence on later startup attempts in the same container lifecycle" - When call template_contains 'elif [ -f "${DATA_DIR}/.prestop-fence-started" ]; then' + When call template_contains 'if [ -f "${DATA_DIR}/.prestop-fence-started" ]; then' The status should be success The output should include ".prestop-fence-started" End diff --git a/addons/mariadb/scripts-ut-spec/replication_primary_write_commit_linearization_spec.sh b/addons/mariadb/scripts-ut-spec/replication_primary_write_commit_linearization_spec.sh index 20c0d1c7ef..ca8736e221 100644 --- a/addons/mariadb/scripts-ut-spec/replication_primary_write_commit_linearization_spec.sh +++ b/addons/mariadb/scripts-ut-spec/replication_primary_write_commit_linearization_spec.sh @@ -39,10 +39,13 @@ Describe "replication primary-write commit linearization" printf '%s\n' '#!/usr/bin/env bash' 'set -u' extract_function_from "$(entrypoint_file)" try_acquire_primary_write_commit_lock extract_function_from "$(entrypoint_file)" release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" persist_primary_write_accept_guard extract_function_from "$(entrypoint_file)" set_primary_read_write extract_function_from "$(prestop_file)" acquire_primary_write_commit_lock_for_prestop cat <<'HARNESS' PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } prestop_watchdog_log() { trace_event "accept:$*"; } prestop_log() { trace_event "prestop:$*"; } @@ -72,6 +75,11 @@ authoritative_primary_write_commit() { printf '%s\n' OFF > "${DATA_DIR}/global" trace_event syncer-authority-commit-pass } +authoritative_primary_write_publish() { + printf '%s\n' "$2" > "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" + rm -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" + trace_event syncer-publication-commit-pass +} run_accept() { set_primary_read_write linearization require-dcs-primary @@ -132,6 +140,257 @@ HARNESS return "${rc}" } + run_authoritative_success_then_publish_markers() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + trace="${work_dir}/trace" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}" + printf '%s\n' NO_LOCK_NO_ADMIN > "${data_dir}/global" + touch "${data_dir}/local-locked" "${data_dir}/remote-locked" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function_from "$(entrypoint_file)" try_acquire_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" persist_primary_write_accept_guard + extract_function_from "$(entrypoint_file)" set_primary_read_write + cat <<'HARNESS' +PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" +trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } +prestop_watchdog_log() { trace_event "accept:$*"; } +mark_replication_pending() { + rm -f "${DATA_DIR}/.primary-read-write-ready" "${DATA_DIR}/.replication-ready" + touch "${DATA_DIR}/.replication-pending" +} +mark_replication_ready() { + touch "${DATA_DIR}/.replication-ready" + rm -f "${DATA_DIR}/.replication-pending" + trace_event replication-ready-published-by-caller +} +read_only_is_fail_closed() { return 0; } +primary_write_gates_ready() { return 0; } +rollback_locked_primary_accept() { trace_event unexpected-rollback; return 2; } +rollback_fenced_primary_accept() { trace_event unexpected-rollback; return 0; } +unlock_local_root_writes() { rm -f "${DATA_DIR}/local-locked"; } +unlock_remote_root_writes() { rm -f "${DATA_DIR}/remote-locked"; } +authoritative_primary_write_commit() { + printf '%s\n' OFF > "${DATA_DIR}/global" + trace_event syncer-terminal-success-received + return 0 +} +authoritative_primary_write_publish() { + printf '%s\n' "$2" > "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" + rm -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" + trace_event syncer-publication-commit-pass +} + +set_primary_read_write caller-publication require-dcs-primary +[ "$?" -eq 0 ] +[ -f "${DATA_DIR}/.primary-read-write-ready" ] +[ -f "${DATA_DIR}/.replication-ready" ] +[ ! -f "${DATA_DIR}/.replication-pending" ] +[ ! -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] +syncer_line="$(grep -n '^syncer-terminal-success-received$' "${TRACE_FILE}" | cut -d: -f1)" +replication_line="$(grep -n '^replication-ready-published-by-caller$' "${TRACE_FILE}" | cut -d: -f1)" +[ -n "${syncer_line}" ] +[ -n "${replication_line}" ] +[ "${syncer_line}" -lt "${replication_line}" ] +cat "${TRACE_FILE}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_authoritative_failure_after_writer_open_rolls_back() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + trace="${work_dir}/trace" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}" + printf '%s\n' NO_LOCK_NO_ADMIN > "${data_dir}/global" + touch "${data_dir}/local-locked" "${data_dir}/remote-locked" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function_from "$(entrypoint_file)" try_acquire_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" force_release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" persist_primary_write_accept_guard + extract_function_from "$(entrypoint_file)" rollback_ambiguous_primary_publish + extract_function_from "$(entrypoint_file)" set_primary_read_write + cat <<'HARNESS' +PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_PUBLICATION_LOCK_DIR="${DATA_DIR}/.primary-write-publication-lock" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" +trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } +prestop_watchdog_log() { trace_event "accept:$*"; } +mark_replication_pending() { + rm -f "${DATA_DIR}/.primary-read-write-ready" "${DATA_DIR}/.replication-ready" + touch "${DATA_DIR}/.replication-pending" +} +mark_replication_ready() { + touch "${DATA_DIR}/.replication-ready" + rm -f "${DATA_DIR}/.replication-pending" + trace_event unexpected-ready-publication +} +read_only_is_fail_closed() { return 0; } +primary_write_gates_ready() { return 0; } +unlock_local_root_writes() { rm -f "${DATA_DIR}/local-locked"; } +unlock_remote_root_writes() { rm -f "${DATA_DIR}/remote-locked"; } +authoritative_primary_write_commit() { + printf '%s\n' OFF > "${DATA_DIR}/global" + trace_event syncer-opened-writers-before-error + return 1 +} +rollback_fenced_primary_accept() { + mark_replication_pending + printf '%s\n' NO_LOCK_NO_ADMIN > "${DATA_DIR}/global" + touch "${DATA_DIR}/local-locked" "${DATA_DIR}/remote-locked" + trace_event caller-rollback-strongest-fence + return 0 +} +rollback_locked_primary_accept() { + rollback_fenced_primary_accept "$1" "$2" + release_primary_write_commit_lock + return 2 +} +acquire_primary_write_publication_lock_for_rollback() { + mkdir "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" + trace_event caller-acquired-publication-lock +} +release_primary_write_publication_lock() { + rmdir "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" + trace_event caller-released-publication-lock +} + +accept_rc=0 +set_primary_read_write caller-rollback require-dcs-primary || accept_rc=$? +failure=0 +[ "${accept_rc}" -eq 2 ] || failure=1 +[ "$(cat "${DATA_DIR}/global")" = NO_LOCK_NO_ADMIN ] || failure=1 +[ -f "${DATA_DIR}/local-locked" ] || failure=1 +[ -f "${DATA_DIR}/remote-locked" ] || failure=1 +[ -f "${DATA_DIR}/.replication-pending" ] || failure=1 +[ -f "${DATA_DIR}/.primary-write-accept-pending" ] || failure=1 +[ ! -f "${DATA_DIR}/.replication-ready" ] || failure=1 +[ ! -f "${DATA_DIR}/.primary-read-write-ready" ] || failure=1 +cat "${TRACE_FILE}" +exit "${failure}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_authority_drift_after_marker_stage_before_publication_commit() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + trace="${work_dir}/trace" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}" + printf '%s\n' NO_LOCK_NO_ADMIN > "${data_dir}/global" + printf '%s\n' primary > "${data_dir}/authority" + touch "${data_dir}/local-locked" "${data_dir}/remote-locked" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function_from "$(entrypoint_file)" try_acquire_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" force_release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" force_release_primary_write_publication_lock_after_commit_receipt + extract_function_from "$(entrypoint_file)" primary_write_publication_receipt_matches + extract_function_from "$(entrypoint_file)" recover_committed_primary_write_publication + extract_function_from "$(entrypoint_file)" persist_primary_write_accept_guard + extract_function_from "$(entrypoint_file)" rollback_ambiguous_primary_publish + extract_function_from "$(entrypoint_file)" set_primary_read_write + cat <<'HARNESS' +PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_PUBLICATION_LOCK_DIR="${DATA_DIR}/.primary-write-publication-lock" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" +trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } +prestop_watchdog_log() { trace_event "accept:$*"; } +mark_replication_pending() { + rm -f "${DATA_DIR}/.primary-read-write-ready" "${DATA_DIR}/.replication-ready" + touch "${DATA_DIR}/.replication-pending" +} +mark_replication_ready() { + touch "${DATA_DIR}/.replication-ready" + rm -f "${DATA_DIR}/.replication-pending" + trace_event caller-markers-staged +} +read_only_is_fail_closed() { return 0; } +primary_write_gates_ready() { return 0; } +unlock_local_root_writes() { rm -f "${DATA_DIR}/local-locked"; } +unlock_remote_root_writes() { rm -f "${DATA_DIR}/remote-locked"; } +authoritative_primary_write_commit() { + printf '%s\n' OFF > "${DATA_DIR}/global" + trace_event syncer-writer-open-committed + return 0 +} +authoritative_primary_write_publish() { + # Model RunCycle winning the HA mutex after the writer-open receipt but + # before publication. The production finalize operation must refresh + # authority and reject without deleting the durable guard. + printf '%s\n' secondary > "${DATA_DIR}/authority" + trace_event run-cycle-demoted-before-publication-commit + return 1 +} +rollback_fenced_primary_accept() { + mark_replication_pending + printf '%s\n' NO_LOCK_NO_ADMIN > "${DATA_DIR}/global" + touch "${DATA_DIR}/local-locked" "${DATA_DIR}/remote-locked" + trace_event caller-rollback-strongest-fence + return 0 +} +rollback_locked_primary_accept() { + rollback_fenced_primary_accept "$1" "$2" + release_primary_write_commit_lock + return 2 +} +acquire_primary_write_publication_lock_for_rollback() { + mkdir "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" + trace_event caller-acquired-publication-lock +} +release_primary_write_publication_lock() { + rmdir "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" + trace_event caller-released-publication-lock +} + +accept_rc=0 +set_primary_read_write post-marker-drift require-dcs-primary || accept_rc=$? +failure=0 +[ "${accept_rc}" -eq 2 ] || failure=1 +[ "$(cat "${DATA_DIR}/authority")" = secondary ] || failure=1 +[ "$(cat "${DATA_DIR}/global")" = NO_LOCK_NO_ADMIN ] || failure=1 +[ -f "${DATA_DIR}/local-locked" ] || failure=1 +[ -f "${DATA_DIR}/remote-locked" ] || failure=1 +[ -f "${DATA_DIR}/.replication-pending" ] || failure=1 +[ -f "${DATA_DIR}/.primary-write-accept-pending" ] || failure=1 +[ ! -f "${DATA_DIR}/.replication-ready" ] || failure=1 +[ ! -f "${DATA_DIR}/.primary-read-write-ready" ] || failure=1 +cat "${TRACE_FILE}" +exit "${failure}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + run_accept_held_past_prestop_lock_budget() { work_dir="$(mktemp -d)" harness="${work_dir}/harness.sh" @@ -145,10 +404,13 @@ HARNESS printf '%s\n' '#!/usr/bin/env bash' 'set -u' extract_function_from "$(entrypoint_file)" try_acquire_primary_write_commit_lock extract_function_from "$(entrypoint_file)" release_primary_write_commit_lock + extract_function_from "$(entrypoint_file)" persist_primary_write_accept_guard extract_function_from "$(entrypoint_file)" set_primary_read_write extract_function_from "$(prestop_file)" acquire_primary_write_commit_lock_for_prestop cat <<'HARNESS' PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" trace_event() { printf '%s\n' "$1" >> "${TRACE_FILE}"; } prestop_watchdog_log() { trace_event "accept:$*"; } prestop_log() { trace_event "prestop:$*"; } @@ -178,6 +440,7 @@ authoritative_primary_write_commit() { printf '%s\n' OFF > "${DATA_DIR}/global" trace_event syncer-authority-commit-pass } +query_local_syncer_role() { printf '%s\n' primary; } run_accept() { accept_rc=0 @@ -229,15 +492,172 @@ HARNESS return "${rc}" } - startup_clears_only_stale_commit_lock() { + startup_clears_only_stale_primary_write_locks() { awk ' /^if \[ ! -f "\$\{LIFECYCLE_MARKER\}" \]; then$/ { startup = 1 } - startup && /^elif \[ -f "\$\{DATA_DIR\}\/\.prestop-fence-started" \]; then$/ { startup = 0 } - startup && index($0, "rm -rf \"${PRIMARY_WRITE_COMMIT_LOCK_DIR}\"") { print; found++ } - END { exit(found == 1 ? 0 : 1) } + startup && /^rm -f "\$\{LIFECYCLE_PENDING_MARKER\}"/ { startup = 0 } + startup && index($0, "\"${PRIMARY_WRITE_COMMIT_LOCK_DIR}\"") { print; commit++ } + startup && index($0, "\"${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}\"") { print; publication++ } + END { exit(commit >= 1 && publication >= 1 ? 0 : 1) } ' "$(entrypoint_file)" } + extract_startup_lifecycle_gate() { + awk ' + /^LIFECYCLE_MARKER="\/tmp\/\.mariadb-startup-lifecycle"$/ { inside = 1 } + inside && /^if \[ -f "\$\{DATA_DIR\}\/\.prestop-fence-started" \]; then$/ { exit } + inside { + print + } + END { if (!inside) exit 1 } + ' "$(entrypoint_file)" + } + + emit_startup_lifecycle_gate_function() { + printf '%s\n' 'run_startup_lifecycle_gate() {' + extract_startup_lifecycle_gate \ + | sed 's#LIFECYCLE_MARKER="/tmp/.mariadb-startup-lifecycle"#LIFECYCLE_MARKER="${TEST_LIFECYCLE_MARKER}"#' + printf '%s\n' '}' + } + + run_lifecycle_marker_touch_failure_preserves_live_publication_owner() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + data_dir="${work_dir}/data" + lifecycle_marker="${work_dir}/startup-lifecycle" + mkdir -p \ + "${data_dir}/.primary-write-commit-lock" \ + "${data_dir}/.primary-write-publication-lock" + printf '%s\n' live-commit-owner \ + > "${data_dir}/.primary-write-commit-lock/owner" + printf '%s\n' live-publication-owner \ + > "${data_dir}/.primary-write-publication-lock/owner" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + cat <<'HARNESS' +touch() { + if [ "$1" = "${TEST_LIFECYCLE_MARKER}.pending" ]; then + printf '%s\n' pending-lifecycle-marker-touch-failed + return 1 + fi + command touch "$@" +} +HARNESS + emit_startup_lifecycle_gate_function + cat <<'HARNESS' +( run_startup_lifecycle_gate ) 2>&1 +gate_rc=$? +printf 'entrypoint_rc=%s lifecycle_marker_exists=%s commit_lock_exists=%s publication_lock_exists=%s\n' \ + "${gate_rc}" \ + "$([ -f "${TEST_LIFECYCLE_MARKER}" ] && printf true || printf false)" \ + "$([ -f "${DATA_DIR}/.primary-write-commit-lock/owner" ] && printf true || printf false)" \ + "$([ -f "${DATA_DIR}/.primary-write-publication-lock/owner" ] && printf true || printf false)" +[ "${gate_rc}" -ne 0 ] +[ ! -f "${TEST_LIFECYCLE_MARKER}" ] +[ "$(cat "${DATA_DIR}/.primary-write-commit-lock/owner")" = live-commit-owner ] +[ "$(cat "${DATA_DIR}/.primary-write-publication-lock/owner")" = live-publication-owner ] +HARNESS + } > "${harness}" + + TEST_LIFECYCLE_MARKER="${lifecycle_marker}" DATA_DIR="${data_dir}" \ + bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_precleanup_crash_then_retry_converges_stale_lock_cleanup() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + data_dir="${work_dir}/data" + lifecycle_marker="${work_dir}/startup-lifecycle" + mkdir -p \ + "${data_dir}/.primary-write-commit-lock" \ + "${data_dir}/.primary-write-publication-lock" + printf '%s\n' stale-commit-owner \ + > "${data_dir}/.primary-write-commit-lock/owner" + printf '%s\n' stale-publication-owner \ + > "${data_dir}/.primary-write-publication-lock/owner" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + cat <<'HARNESS' +rm() { + if [ "${CRASH_BEFORE_CLEANUP:-0}" -eq 1 ]; then + printf '%s\n' injected-process-death-before-cleanup + exit 99 + fi + command rm "$@" +} +HARNESS + emit_startup_lifecycle_gate_function + cat <<'HARNESS' +( CRASH_BEFORE_CLEANUP=1 run_startup_lifecycle_gate ) 2>&1 +first_rc=$? +pending_after_crash=false +[ -f "${TEST_LIFECYCLE_MARKER}.pending" ] && pending_after_crash=true +( CRASH_BEFORE_CLEANUP=0 run_startup_lifecycle_gate ) 2>&1 +retry_rc=$? +printf 'first_rc=%s retry_rc=%s pending_after_crash=%s lifecycle_marker_exists=%s commit_lock_exists=%s publication_lock_exists=%s\n' \ + "${first_rc}" "${retry_rc}" "${pending_after_crash}" \ + "$([ -f "${TEST_LIFECYCLE_MARKER}" ] && printf true || printf false)" \ + "$([ -d "${DATA_DIR}/.primary-write-commit-lock" ] && printf true || printf false)" \ + "$([ -d "${DATA_DIR}/.primary-write-publication-lock" ] && printf true || printf false)" +[ "${first_rc}" -eq 99 ] +[ "${pending_after_crash}" = true ] +[ "${retry_rc}" -eq 0 ] +[ -f "${TEST_LIFECYCLE_MARKER}" ] +[ ! -f "${TEST_LIFECYCLE_MARKER}.pending" ] +[ ! -d "${DATA_DIR}/.primary-write-commit-lock" ] +[ ! -d "${DATA_DIR}/.primary-write-publication-lock" ] +HARNESS + } > "${harness}" + + TEST_LIFECYCLE_MARKER="${lifecycle_marker}" DATA_DIR="${data_dir}" \ + bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_committed_lifecycle_preserves_same_process_live_owner() { + work_dir="$(mktemp -d)" + harness="${work_dir}/harness.sh" + data_dir="${work_dir}/data" + lifecycle_marker="${work_dir}/startup-lifecycle" + mkdir -p \ + "${data_dir}/.primary-write-commit-lock" \ + "${data_dir}/.primary-write-publication-lock" + touch "${lifecycle_marker}" + printf '%s\n' live-commit-owner \ + > "${data_dir}/.primary-write-commit-lock/owner" + printf '%s\n' live-publication-owner \ + > "${data_dir}/.primary-write-publication-lock/owner" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + emit_startup_lifecycle_gate_function + cat <<'HARNESS' +run_startup_lifecycle_gate +gate_rc=$? +printf 'entrypoint_rc=%s commit_lock_exists=%s publication_lock_exists=%s\n' \ + "${gate_rc}" \ + "$([ -f "${DATA_DIR}/.primary-write-commit-lock/owner" ] && printf true || printf false)" \ + "$([ -f "${DATA_DIR}/.primary-write-publication-lock/owner" ] && printf true || printf false)" +[ "${gate_rc}" -eq 0 ] +[ "$(cat "${DATA_DIR}/.primary-write-commit-lock/owner")" = live-commit-owner ] +[ "$(cat "${DATA_DIR}/.primary-write-publication-lock/owner")" = live-publication-owner ] +HARNESS + } > "${harness}" + + TEST_LIFECYCLE_MARKER="${lifecycle_marker}" DATA_DIR="${data_dir}" \ + bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + prestop_acquires_commit_lock_before_marker() { lock_line="$(grep -n '^if ! acquire_primary_write_commit_lock_for_prestop; then$' "$(prestop_file)" | cut -d: -f1)" marker_line="$(grep -n '^touch "${DATA_DIR}/.prestop-fence-started"' "$(prestop_file)" | cut -d: -f1)" @@ -254,6 +674,29 @@ HARNESS The output should not include "unexpected-rollback" End + It "publishes primary and replication readiness only after terminal syncer success" + When call run_authoritative_success_then_publish_markers + The status should be success + The output should include "syncer-terminal-success-received" + The output should include "replication-ready-published-by-caller" + The output should not include "unexpected-rollback" + End + + It "restores the strongest fence when syncer errors after opening writers" + When call run_authoritative_failure_after_writer_open_rolls_back + The status should be success + The output should include "syncer-opened-writers-before-error" + The output should include "caller-rollback-strongest-fence" + The output should not include "unexpected-ready-publication" + End + + It "rejects authority drift after marker staging before the mutex-linearized publication commit" + When call run_authority_drift_after_marker_stage_before_publication_commit + The status should be success + The output should include "run-cycle-demoted-before-publication-commit" + The output should include "caller-rollback-strongest-fence" + End + It "does not let an accept publish ready after preStop exhausts its commit-lock budget" When call run_accept_held_past_prestop_lock_budget The status should be success @@ -261,10 +704,39 @@ HARNESS The output should not include "replication-ready-published" End - It "clears a stale commit owner only on a fresh container lifecycle" - When call startup_clears_only_stale_commit_lock + It "clears stale commit and publication owners only on a fresh container lifecycle" + When call startup_clears_only_stale_primary_write_locks The status should be success The output should include 'PRIMARY_WRITE_COMMIT_LOCK_DIR' + The output should include 'PRIMARY_WRITE_PUBLICATION_LOCK_DIR' + End + + It "fails closed without clearing a live publication owner when lifecycle identity cannot be persisted" + When call run_lifecycle_marker_touch_failure_preserves_live_publication_owner + The status should be success + The output should include "entrypoint_rc=1" + The output should include "lifecycle_marker_exists=false" + The output should include "commit_lock_exists=true" + The output should include "publication_lock_exists=true" + End + + It "retries stale lock cleanup after process death before lifecycle completion" + When call run_precleanup_crash_then_retry_converges_stale_lock_cleanup + The status should be success + The output should include "first_rc=99" + The output should include "retry_rc=0" + The output should include "pending_after_crash=true" + The output should include "lifecycle_marker_exists=true" + The output should include "commit_lock_exists=false" + The output should include "publication_lock_exists=false" + End + + It "does not clear a same-process live owner after lifecycle completion" + When call run_committed_lifecycle_preserves_same_process_live_owner + The status should be success + The output should include "entrypoint_rc=0" + The output should include "commit_lock_exists=true" + The output should include "publication_lock_exists=true" End It "makes preStop own the commit lock before publishing its fence marker" diff --git a/addons/mariadb/scripts-ut-spec/replication_roleprobe_spec.sh b/addons/mariadb/scripts-ut-spec/replication_roleprobe_spec.sh index 3d9dd63a1c..b3e05eb169 100644 --- a/addons/mariadb/scripts-ut-spec/replication_roleprobe_spec.sh +++ b/addons/mariadb/scripts-ut-spec/replication_roleprobe_spec.sh @@ -585,6 +585,22 @@ Last_SQL_Errno: 0" End End + Context "when caller marker files are staged behind a durable accept guard" + setup_primary_accept_pending() { + touch "${TEST_DIR}/.replication-ready" + touch "${TEST_DIR}/.sql-listener-ready" + touch "${TEST_DIR}/.primary-read-write-ready" + touch "${TEST_DIR}/.primary-write-accept-pending" + } + Before "setup_primary_accept_pending" + + It "does not publish primary until the exact caller clears the accept guard" + When call check_role + The status should be failure + The output should eq "initializing" + End + End + Context "when sql listener readiness is required but marker is missing" setup_primary_without_listener_marker() { export MARIADB_ROLEPROBE_REQUIRE_SQL_LISTENER_READY="true" diff --git a/addons/mariadb/scripts-ut-spec/replication_runtime_role_promotion_race_spec.sh b/addons/mariadb/scripts-ut-spec/replication_runtime_role_promotion_race_spec.sh index 0ee476361b..6bc75d1b9c 100644 --- a/addons/mariadb/scripts-ut-spec/replication_runtime_role_promotion_race_spec.sh +++ b/addons/mariadb/scripts-ut-spec/replication_runtime_role_promotion_race_spec.sh @@ -50,6 +50,15 @@ query_local_syncer_role() { printf '%s\n' primary fi } +local_primary_role_published() { + return 1 +} +read_only_is_writable() { + return 1 +} +read_only_is_fail_closed() { + grep -q '^read-only-on$' "${TRACE_FILE}" +} lock_remote_root_writes() { trace_event remote-root-fenced } @@ -113,6 +122,201 @@ HARNESS return "${rc}" } + run_promotion_before_first_replica_fence() { + work_dir="$(mktemp -d)" + harness="${work_dir}/early-race-harness.sh" + trace="${work_dir}/trace" + role_counter="${work_dir}/role-counter" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}" + printf '0\n' > "${role_counter}" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function set_replica_read_only + extract_function replica_lock_abort_if_syncer_primary + extract_function accept_syncer_primary_promotion_from_replica_path + extract_function reconcile_sql_listener_for_syncer_secondary_once + cat <<'HARNESS' +GLOBAL_READ_ONLY=0 +trace_event() { + printf '%s\n' "$1" >> "${TRACE_FILE}" +} +query_local_syncer_role() { + local call + call=$(( $(cat "${ROLE_COUNTER}") + 1 )) + printf '%s\n' "${call}" > "${ROLE_COUNTER}" + if [ "${call}" -eq 1 ]; then + printf '%s\n' secondary + else + printf '%s\n' primary + fi +} +local_primary_role_published() { + return 1 +} +read_only_is_writable() { + [ "${GLOBAL_READ_ONLY}" -eq 0 ] +} +read_only_is_fail_closed() { + [ "${GLOBAL_READ_ONLY}" -eq 1 ] +} +set_fail_closed_read_only() { + GLOBAL_READ_ONLY=1 + trace_event read-only-on +} +prestop_watchdog_log() { + : +} +expose_sql_listener_for_primary_role() { + if [ "${GLOBAL_READ_ONLY}" -eq 1 ]; then + trace_event full-primary-accept-fenced + else + trace_event full-primary-accept-unfenced + fi +} +mark_replication_pending() { + trace_event replication-pending +} +query_slave_status_verbose() { + return 1 +} +slave_status_is_healthy() { + return 1 +} +publish_replica_after_rejoin_ready() { + return 1 +} +configure_replication_from_primary_service_once() { + return 1 +} + +reconcile_sql_listener_for_syncer_secondary_once + +cat "${TRACE_FILE}" +read_only_line="$(grep -n '^read-only-on$' "${TRACE_FILE}" | cut -d: -f1)" +full_accept_line="$(grep -n '^full-primary-accept-fenced$' "${TRACE_FILE}" | cut -d: -f1)" +[ -n "${read_only_line}" ] +[ -n "${full_accept_line}" ] +! grep -q '^full-primary-accept-unfenced$' "${TRACE_FILE}" +[ "${read_only_line}" -lt "${full_accept_line}" ] +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" ROLE_COUNTER="${role_counter}" DATA_DIR="${data_dir}" \ + bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_pre_dcs_secondary_defers_writable_pod0() { + work_dir="$(mktemp -d)" + harness="${work_dir}/pre-dcs-secondary-harness.sh" + trace="${work_dir}/trace" + data_dir="${work_dir}/data" + mkdir -p "${data_dir}" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function reconcile_sql_listener_for_syncer_secondary_once + cat <<'HARNESS' +trace_event() { + printf '%s\n' "$1" >> "${TRACE_FILE}" +} +query_local_syncer_role() { + printf '%s\n' secondary +} +local_primary_role_published() { + return 0 +} +read_only_is_writable() { + return 0 +} +set_replica_read_only() { + trace_event read-only-on + return 0 +} +prestop_watchdog_log() { + trace_event "log:$*" +} +mark_replication_pending() { + trace_event replication-pending +} +query_slave_status_verbose() { + return 1 +} +slave_status_is_healthy() { + return 1 +} +publish_replica_after_rejoin_ready() { + return 1 +} +configure_replication_from_primary_service_once() { + return 1 +} +accept_syncer_primary_promotion_from_replica_path() { + return 1 +} + +reconcile_sql_listener_for_syncer_secondary_once +cat "${TRACE_FILE}" +! grep -q '^read-only-on$' "${TRACE_FILE}" +grep -q 'reason=pre-dcs-local-primary-writable' "${TRACE_FILE}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" DATA_DIR="${data_dir}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + + run_pre_accept_fence_failure() { + work_dir="$(mktemp -d)" + harness="${work_dir}/fence-failure-harness.sh" + trace="${work_dir}/trace" + + { + printf '%s\n' '#!/usr/bin/env bash' 'set -u' + extract_function accept_syncer_primary_promotion_from_replica_path + cat <<'HARNESS' +trace_event() { + printf '%s\n' "$1" >> "${TRACE_FILE}" +} +query_local_syncer_role() { + printf '%s\n' primary +} +read_only_is_fail_closed() { + return 1 +} +set_fail_closed_read_only() { + trace_event pre-accept-fence-failed + return 1 +} +prestop_watchdog_log() { + trace_event "log:$*" +} +expose_sql_listener_for_primary_role() { + trace_event unexpected-full-primary-accept +} + +accept_rc=0 +accept_syncer_primary_promotion_from_replica_path fence-failure || accept_rc=$? +cat "${TRACE_FILE}" +[ "${accept_rc}" -eq 3 ] +grep -q '^pre-accept-fence-failed$' "${TRACE_FILE}" +grep -q 'reason=pre-accept-fence-failed fail_closed=false' "${TRACE_FILE}" +! grep -q '^unexpected-full-primary-accept$' "${TRACE_FILE}" +HARNESS + } > "${harness}" + + TRACE_FILE="${trace}" bash "${harness}" + rc=$? + rm -rf "${work_dir}" + return "${rc}" + } + run_business_writer_during_primary_accept() { work_dir="$(mktemp -d)" harness="${work_dir}/business-writer-harness.sh" @@ -129,6 +333,9 @@ trace_event() { query_local_syncer_role() { printf '%s\n' primary } +read_only_is_fail_closed() { + [ "${GLOBAL_READ_ONLY}" -eq 1 ] +} internal_sql() { case "$*" in *"SET GLOBAL read_only = 0;"*) GLOBAL_READ_ONLY=0 ;; @@ -196,6 +403,9 @@ query_local_syncer_role() { printf '%s\n' primary fi } +read_only_is_fail_closed() { + return 0 +} internal_sql() { case "$*" in *"SET GLOBAL read_only = 0;"*) trace_event primary-internal-writable ;; @@ -264,6 +474,29 @@ HARNESS The output should include "full-primary-accept" End + It "installs the fail-closed fence when promotion wins before the first replica lock" + When call run_promotion_before_first_replica_fence + The status should be success + The output should include "read-only-on" + The output should include "full-primary-accept-fenced" + The output should not include "full-primary-accept-unfenced" + End + + It "defers a pre-DCS secondary observation while fresh pod-0 is locally published and explicitly writable" + When call run_pre_dcs_secondary_defers_writable_pod0 + The status should be success + The output should include "pre-dcs-local-primary-writable" + The output should not include "read-only-on" + End + + It "fails closed and skips full acceptance when the pre-accept fence cannot be installed" + When call run_pre_accept_fence_failure + The status should be success + The output should include "pre-accept-fence-failed" + The output should include "fail_closed=false" + The output should not include "unexpected-full-primary-accept" + End + It "keeps an ordinary business writer fenced before full-primary acceptance" When call run_business_writer_during_primary_accept The status should be success diff --git a/addons/mariadb/scripts-ut-spec/semisync_rejoin_fence_template_spec.sh b/addons/mariadb/scripts-ut-spec/semisync_rejoin_fence_template_spec.sh index f420fbe0d3..ac831601a7 100644 --- a/addons/mariadb/scripts-ut-spec/semisync_rejoin_fence_template_spec.sh +++ b/addons/mariadb/scripts-ut-spec/semisync_rejoin_fence_template_spec.sh @@ -110,6 +110,83 @@ Describe "cmpd-replication.yaml rejoin fence template" [ "${primary_query_line}" -lt "${reconcile_line}" ] } + guarded_primary_accept_retry_precedes_published_role_query() { + awk ' + index($0, "reconcile_sql_listener_for_syncer_primary_once() {") { fn = 1 } + fn && index($0, "PRIMARY_WRITE_ACCEPT_PENDING_FILE") { pending_guard = NR } + fn && pending_guard && index($0, "master.info") { guard = NR } + fn && index($0, "runtime-primary-accept-retry-begin") { trace = NR } + fn && index($0, "expose_sql_listener_for_primary_role \"syncer-primary-accept-pending-retry\" \"fenced-promotion\"") { retry = NR } + fn && index($0, "role=\"$(query_local_syncer_role || true)\"") { role = NR; exit } + END { exit(guard && trace && retry && role && guard < trace && trace < retry && retry < role ? 0 : 1) } + ' "$(template_file)" + } + + guarded_primary_accept_retry_uses_authoritative_path() { + tmpdir="$(mktemp -d)" || return 1 + DATA_DIR="${tmpdir}" + PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" + touch "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" || return 1 + + eval "$(extract_function_definition reconcile_sql_listener_for_syncer_primary_once)" + + trace="" + exposed="" + role_queries=0 + prestop_watchdog_log() { trace="${trace}|$*"; } + mariadbd_listen_on_all_interfaces() { return 0; } + expose_sql_listener_for_primary_role() { + exposed="$1|$2" + return 0 + } + query_local_syncer_role() { + role_queries=$((role_queries + 1)) + printf '%s' secondary + } + + reconcile_sql_listener_for_syncer_primary_once + reconcile_rc=$? + guard_after=absent + [ ! -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] || guard_after=present + rm -rf "${tmpdir}" + + [ "${reconcile_rc}" -eq 0 ] && + [ "${exposed}" = "syncer-primary-accept-pending-retry|fenced-promotion" ] && + [ "${role_queries}" -eq 0 ] && + [ "${guard_after}" = present ] && + case "${trace}" in *runtime-primary-accept-retry-complete*) true ;; *) false ;; esac + } + + guarded_primary_accept_rejection_stays_fail_closed() { + tmpdir="$(mktemp -d)" || return 1 + DATA_DIR="${tmpdir}" + PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" + touch "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" || return 1 + + eval "$(extract_function_definition reconcile_sql_listener_for_syncer_primary_once)" + + trace="" + role_queries=0 + prestop_watchdog_log() { trace="${trace}|$*"; } + mariadbd_listen_on_all_interfaces() { return 0; } + expose_sql_listener_for_primary_role() { return 1; } + query_local_syncer_role() { + role_queries=$((role_queries + 1)) + printf '%s' secondary + } + + reconcile_sql_listener_for_syncer_primary_once + reconcile_rc=$? + guard_after=absent + [ ! -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] || guard_after=present + rm -rf "${tmpdir}" + + [ "${reconcile_rc}" -eq 1 ] && + [ "${role_queries}" -eq 0 ] && + [ "${guard_after}" = present ] && + case "${trace}" in *runtime-primary-accept-retry-defer*guard=preserved*) true ;; *) false ;; esac + } + runtime_secondary_follow_starts_io_before_health_cleanup() { begin_line="$(grep -n 'runtime-secondary-follow-configure-begin' "$(template_file)" | head -1 | cut -d: -f1)" io_line="$(awk -v begin="${begin_line}" 'NR > begin && index($0, "START SLAVE IO_THREAD;") { print NR; exit }' "$(template_file)")" @@ -237,7 +314,7 @@ Describe "cmpd-replication.yaml rejoin fence template" End It "keeps the preStop fence on later startup attempts in the same container lifecycle" - When call template_contains 'elif [ -f "${DATA_DIR}/.prestop-fence-started" ]; then' + When call template_contains 'if [ -f "${DATA_DIR}/.prestop-fence-started" ]; then' The status should be success The output should include ".prestop-fence-started" End @@ -430,6 +507,21 @@ Describe "cmpd-replication.yaml rejoin fence template" The status should be success End + It "retries a durable guarded primary accept through the authoritative commit path before requiring published role" + When call guarded_primary_accept_retry_precedes_published_role_query + The status should be success + End + + It "delegates durable guard removal to a successful authoritative retry before querying published role" + When call guarded_primary_accept_retry_uses_authoritative_path + The status should be success + End + + It "keeps a rejected durable accept fail-closed without falling through to published role" + When call guarded_primary_accept_rejection_stays_fail_closed + The status should be success + End + It "exposes syncer-promoted primary only after primary local write access is ready" When call function_contains "reconcile_sql_listener_for_syncer_primary_once" "expose_sql_listener_for_primary_role \"syncer-promoted-primary\"" The status should be success diff --git a/addons/mariadb/scripts/mariadb-image-entrypoint.sh b/addons/mariadb/scripts/mariadb-image-entrypoint.sh new file mode 100644 index 0000000000..78317af32c --- /dev/null +++ b/addons/mariadb/scripts/mariadb-image-entrypoint.sh @@ -0,0 +1,154 @@ +#!/bin/bash +set -euo pipefail + +readonly IMAGE_INIT_IN_PROGRESS_MARKER=".kb-mariadb-image-init-in-progress" +readonly IMAGE_INIT_COMPLETE_MARKER=".kb-mariadb-image-init-complete" +readonly DEFAULT_TEMP_SERVER_START_TIMEOUT_SECONDS=120 + +validate_mariadb_data_dir() { + local data_dir="${1:-}" + case "${data_dir}" in + ""|"/"|"/var"|"/var/lib") + echo "unsafe MariaDB data directory for fresh-init recovery: '${data_dir}'" >&2 + return 1 + ;; + esac + case "${data_dir}" in + /*) ;; + *) + echo "MariaDB data directory must be absolute for fresh-init recovery: '${data_dir}'" >&2 + return 1 + ;; + esac +} + +recover_partial_mariadb_image_init() { + local data_dir="$1" + local in_progress="${data_dir}/${IMAGE_INIT_IN_PROGRESS_MARKER}" + local complete="${data_dir}/${IMAGE_INIT_COMPLETE_MARKER}" + + validate_mariadb_data_dir "${data_dir}" || return 1 + [ -f "${in_progress}" ] || return 0 + + if [ -f "${complete}" ]; then + rm -f "${in_progress}" || return 1 + return 0 + fi + + echo "recovering interrupted addon-owned MariaDB fresh initialization" >&2 + find "${data_dir}" -mindepth 1 -maxdepth 1 \ + ! -name "${IMAGE_INIT_IN_PROGRESS_MARKER}" \ + ! -name "runtime-overrides.cnf" \ + ! -name "runtime-overrides.d" \ + ! -name "log" \ + ! -name "binlog" \ + ! -name "tmp" \ + -exec rm -rf -- {} + || return 1 + rm -f "${in_progress}" || return 1 +} + +validate_temp_server_start_timeout() { + local timeout="$1" + case "${timeout}" in + *[!0-9]*|"") + echo "invalid MariaDB temporary-server startup timeout: '${timeout}'" >&2 + return 1 + ;; + esac + if [ "${timeout}" -lt 31 ] || [ "${timeout}" -gt 300 ]; then + echo "MariaDB temporary-server startup timeout must be between 31 and 300 seconds: '${timeout}'" >&2 + return 1 + fi +} + +prepare_mariadb_image_entrypoint() { + local source_path="$1" + local target_path="$2" + local timeout="${3:-${DEFAULT_TEMP_SERVER_START_TIMEOUT_SECONDS}}" + local tmp_path="${target_path}.tmp.$$" + + validate_temp_server_start_timeout "${timeout}" || return 1 + if [ ! -r "${source_path}" ]; then + echo "MariaDB image entrypoint is missing or unreadable: '${source_path}'" >&2 + return 1 + fi + + rm -f "${tmp_path}" + if ! awk \ + -v timeout="${timeout}" \ + -v in_progress="${IMAGE_INIT_IN_PROGRESS_MARKER}" \ + -v complete="${IMAGE_INIT_COMPLETE_MARKER}" ' + /^[[:space:]]*for i in \{30\.\.0\}; do[[:space:]]*$/ { + timeout_sites++ + match($0, /^[[:space:]]*/) + indent = substr($0, 1, RLENGTH) + print indent "for ((i = " timeout "; i >= 0; i--)); do" + next + } + /^[[:space:]]*docker_mariadb_init "\$@"[[:space:]]*$/ { + init_sites++ + print + match($0, /^[[:space:]]*/) + indent = substr($0, 1, RLENGTH) + print indent "touch \"$DATADIR/" complete "\"" + print indent "rm -f \"$DATADIR/" in_progress "\"" + next + } + { print } + END { + if (timeout_sites != 1 || init_sites != 1) { + print "unsupported MariaDB image entrypoint contract: expected one 30-second loop and one docker_mariadb_init call; found timeout=" timeout_sites ", init=" init_sites > "/dev/stderr" + exit 42 + } + } + ' "${source_path}" > "${tmp_path}"; then + rm -f "${tmp_path}" + return 1 + fi + chmod 0555 "${tmp_path}" || return 1 + mv -f "${tmp_path}" "${target_path}" || return 1 +} + +run_mariadb_image_entrypoint() { + local data_dir="$1" + shift + local source_path="/usr/local/bin/docker-entrypoint.sh" + local target_path="/tmp/kb-mariadb-docker-entrypoint.sh" + local timeout="${MARIADB_TEMP_SERVER_START_TIMEOUT_SECONDS:-${DEFAULT_TEMP_SERVER_START_TIMEOUT_SECONDS}}" + + validate_mariadb_data_dir "${data_dir}" || return 1 + prepare_mariadb_image_entrypoint "${source_path}" "${target_path}" "${timeout}" || return 1 + if [ ! -d "${data_dir}/mysql" ] && [ ! -f "${data_dir}/${IMAGE_INIT_COMPLETE_MARKER}" ]; then + touch "${data_dir}/${IMAGE_INIT_IN_PROGRESS_MARKER}" || return 1 + fi + exec "${target_path}" "$@" +} + +main() { + local mode="${1:-}" + case "${mode}" in + recover) + [ "$#" -eq 2 ] || { + echo "usage: $0 recover DATA_DIR" >&2 + return 2 + } + recover_partial_mariadb_image_init "$2" + ;; + run) + [ "$#" -ge 3 ] || { + echo "usage: $0 run DATA_DIR mariadbd [args...]" >&2 + return 2 + } + shift + run_mariadb_image_entrypoint "$@" + ;; + *) + echo "usage: $0 {recover DATA_DIR|run DATA_DIR mariadbd [args...]}" >&2 + return 2 + ;; + esac +} + +if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then + main "$@" +fi diff --git a/addons/mariadb/scripts/replication-entrypoint.sh b/addons/mariadb/scripts/replication-entrypoint.sh index 97552defbb..ad8e9e0d32 100644 --- a/addons/mariadb/scripts/replication-entrypoint.sh +++ b/addons/mariadb/scripts/replication-entrypoint.sh @@ -79,6 +79,18 @@ if [ -n "${MARIADB_REPLICATION_MODE:-}" ]; then exit 1 fi fi +# The official MariaDB image entrypoint treats the presence of the mysql +# system directory as "initialization complete", although the directory is +# created before the temporary server and account setup have committed. If a +# fresh init crosses the image's old 30-second temporary-server boundary, the +# next container restart otherwise skips account setup permanently. Recover +# only a fresh init that this addon explicitly marked in progress; legacy and +# already-existing datadirs have no marker and are never erased. +if [ ! -x /scripts/mariadb-image-entrypoint.sh ]; then + echo "MariaDB image entrypoint wrapper is missing or not executable; refusing to start" >&2 + exit 1 +fi +/scripts/mariadb-image-entrypoint.sh recover "${DATA_DIR}" # Signal to roleProbe that this pod is initializing — prevents spurious "primary" # reports that would cause KubeBlocks to auto-trigger a switchover. rm -f ${DATA_DIR}/.replication-ready @@ -202,9 +214,24 @@ ROOT_LOCAL=(mariadb "-u${MARIADB_ROOT_USER}" "-p${MARIADB_ROOT_PASSWORD}" -S "${ INTERNAL_LOCAL=(mariadb "-u${MARIADB_INTERNAL_ROOT_USER}" "-p${MARIADB_ROOT_PASSWORD}" -S "${SOCK}" -N -s) LOCAL=("${ROOT_LOCAL[@]}") LIFECYCLE_MARKER="/tmp/.mariadb-startup-lifecycle" +LIFECYCLE_PENDING_MARKER="${LIFECYCLE_MARKER}.pending" PRIMARY_WRITE_COMMIT_LOCK_DIR="${DATA_DIR}/.primary-write-commit-lock" +PRIMARY_WRITE_PUBLICATION_LOCK_DIR="${DATA_DIR}/.primary-write-publication-lock" +PRIMARY_WRITE_ACCEPT_PENDING_FILE="${DATA_DIR}/.primary-write-accept-pending" +PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE="${DATA_DIR}/.primary-write-publication-committed" if [ ! -f "${LIFECYCLE_MARKER}" ]; then - touch "${LIFECYCLE_MARKER}" 2>/dev/null || true + # The pending marker is the retry journal for this container lifecycle. + # Persist it before destructive cleanup, and publish the completion marker + # only after every stale owner is gone. A child-process death anywhere in + # between therefore retries cleanup instead of misclassifying the restart + # as a completed same-lifecycle startup. + if [ ! -f "${LIFECYCLE_PENDING_MARKER}" ]; then + if ! touch "${LIFECYCLE_PENDING_MARKER}" 2>/dev/null || \ + [ ! -f "${LIFECYCLE_PENDING_MARKER}" ]; then + echo "failed to persist pending startup lifecycle identity; refusing stale primary-write lock cleanup" >&2 + exit 1 + fi + fi if [ -f "${DATA_DIR}/.prestop-fence-started" ] || \ [ -f "${DATA_DIR}/.prestop-fence-complete" ] || \ [ -f "${DATA_DIR}/.prestop-fence-failed" ]; then @@ -223,18 +250,36 @@ if [ ! -f "${LIFECYCLE_MARKER}" ]; then # publish markers describe the old process, not the new one. # Clear them together with the stale preStop fence so startup # must re-prove listener exposure and role readiness. - rm -f ${DATA_DIR}/.prestop-fence-started \ - ${DATA_DIR}/.prestop-fence-complete \ - ${DATA_DIR}/.prestop-fence-failed \ - ${DATA_DIR}/.prestop-fence-watchdog-active \ - ${DATA_DIR}/.sql-listener-ready \ - ${DATA_DIR}/.primary-read-write-ready \ - ${DATA_DIR}/.replication-ready + if ! rm -f ${DATA_DIR}/.prestop-fence-started \ + ${DATA_DIR}/.prestop-fence-complete \ + ${DATA_DIR}/.prestop-fence-failed \ + ${DATA_DIR}/.prestop-fence-watchdog-active \ + ${DATA_DIR}/.sql-listener-ready \ + ${DATA_DIR}/.primary-read-write-ready \ + ${DATA_DIR}/.replication-ready; then + echo "failed to clear stale lifecycle publication markers" >&2 + exit 1 + fi # The lock directory belongs to the previous container process. A live # container never removes another invocation's lock; only this new-lifecycle # branch is allowed to clear a stale owner after process death. - rm -rf "${PRIMARY_WRITE_COMMIT_LOCK_DIR}" -elif [ -f "${DATA_DIR}/.prestop-fence-started" ]; then + if ! rm -rf "${PRIMARY_WRITE_COMMIT_LOCK_DIR}" \ + "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" || \ + ! rm -f "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" || \ + [ -e "${PRIMARY_WRITE_COMMIT_LOCK_DIR}" ] || \ + [ -e "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" ] || \ + [ -e "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" ]; then + echo "failed to clear stale primary-write lock owners" >&2 + exit 1 + fi + if ! touch "${LIFECYCLE_MARKER}" 2>/dev/null || \ + [ ! -f "${LIFECYCLE_MARKER}" ]; then + echo "failed to commit startup lifecycle identity after stale lock cleanup" >&2 + exit 1 + fi +fi +rm -f "${LIFECYCLE_PENDING_MARKER}" 2>/dev/null || true +if [ -f "${DATA_DIR}/.prestop-fence-started" ]; then mkdir -p ${DATA_DIR}/log 2>/dev/null || true { printf 'timestamp=%s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" @@ -270,6 +315,16 @@ read_only_is_fail_closed() { esac return 1 } +read_only_is_writable() { + local value + value="$(read_only_value || true)" + case "${value}" in + 0|OFF) + return 0 + ;; + esac + return 1 +} read_only_is_strongest_fail_closed() { local value value="$(read_only_value || true)" @@ -513,6 +568,69 @@ force_release_primary_write_commit_lock() { # unexpected payload made the owner-private directory non-empty. rm -rf "${PRIMARY_WRITE_COMMIT_LOCK_DIR}" 2>/dev/null } +try_acquire_primary_write_publication_lock() { + mkdir "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" 2>/dev/null +} +acquire_primary_write_publication_lock_for_rollback() { + local attempt=0 + # The syncer endpoint has a 4s operation deadline and the CLI has a bounded + # terminal wait. A 7s recovery budget therefore covers a live handler without + # approaching the kbagent 60s clamp. + while [ "${attempt}" -lt 70 ]; do + if try_acquire_primary_write_publication_lock; then + prestop_watchdog_log "primary-write-publication-lock rc=0 owner=caller-rollback attempts=${attempt}" + return 0 + fi + attempt=$((attempt + 1)) + sleep 0.1 + done + prestop_watchdog_log "primary-write-publication-lock rc=1 owner=live-handler attempts=${attempt} fail_closed=false" + return 1 +} +release_primary_write_publication_lock() { + rmdir "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" 2>/dev/null +} +force_release_primary_write_publication_lock_after_commit_receipt() { + # The exact durable receipt proves the handler has crossed its atomic + # publication commit point. Its remaining tail may only clean this lock and + # return, so an ambiguous caller may safely finish stale-lock cleanup. + rm -rf "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" 2>/dev/null && \ + [ ! -e "${PRIMARY_WRITE_PUBLICATION_LOCK_DIR}" ] +} +primary_write_publication_receipt_matches() { + local expected_token="$1" + local actual_token + [ -n "${expected_token}" ] || return 1 + [ ! -e "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] || return 1 + [ -f "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" ] || return 1 + actual_token="$(cat "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" 2>/dev/null)" || return 1 + [ "${actual_token}" = "${expected_token}" ] +} +recover_committed_primary_write_publication() { + local expected_token="$1" + local label="${2:-fenced-primary-accept}" + primary_write_publication_receipt_matches "${expected_token}" || return 1 + if ! force_release_primary_write_publication_lock_after_commit_receipt; then + prestop_watchdog_log "primary-write-publish label=${label} rc=3 reason=committed-receipt-lock-cleanup-failed fail_closed=false" + return 3 + fi + prestop_watchdog_log "primary-write-publish label=${label} rc=0 authority=fresh-lease-cas publication=durable-exact-receipt recovery=ambiguous-response" + return 0 +} +persist_primary_write_accept_guard() { + local transaction_token="$1" + local tmp_file="${PRIMARY_WRITE_ACCEPT_PENDING_FILE}.tmp.$$" + [ -n "${transaction_token}" ] || return 1 + if ! (umask 077 && printf '%s\n' "${transaction_token}" > "${tmp_file}"); then + rm -f "${tmp_file}" 2>/dev/null || true + return 1 + fi + if ! mv -f "${tmp_file}" "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}"; then + rm -f "${tmp_file}" 2>/dev/null || true + return 1 + fi + [ "$(cat "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" 2>/dev/null)" = "${transaction_token}" ] +} authoritative_primary_write_commit() { local label="${1:-primary-write-commit}" local rc @@ -530,6 +648,32 @@ authoritative_primary_write_commit() { prestop_watchdog_log "primary-write-commit label=${label} rc=${rc} authority=rejected" return "${rc}" } +authoritative_primary_write_publish() { + local label="${1:-primary-write-publish}" + local expected_token="${2:-}" + local rc + if [ ! -x /tools/syncerctl ]; then + prestop_watchdog_log "primary-write-publish label=${label} rc=1 reason=syncerctl-missing" + return 1 + fi + timeout 5 /tools/syncerctl --host 127.0.0.1 --port 3601 primary-write-publish \ + >> "${DATA_DIR}/log/prestop-watchdog.log" 2>&1 + rc=$? + if primary_write_publication_receipt_matches "${expected_token}"; then + if ! force_release_primary_write_publication_lock_after_commit_receipt; then + prestop_watchdog_log "primary-write-publish label=${label} rc=3 reason=committed-receipt-lock-cleanup-failed command_rc=${rc} fail_closed=false" + return 3 + fi + prestop_watchdog_log "primary-write-publish label=${label} rc=0 authority=fresh-lease-cas publication=durable-exact-receipt command_rc=${rc}" + return 0 + fi + if [ "${rc}" -eq 0 ]; then + prestop_watchdog_log "primary-write-publish label=${label} rc=1 reason=exact-durable-receipt-missing" + return 1 + fi + prestop_watchdog_log "primary-write-publish label=${label} rc=${rc} authority=rejected" + return "${rc}" +} rollback_locked_primary_accept() { local label="${1:-fenced-primary-accept}" local reason="${2:-unknown}" @@ -546,10 +690,64 @@ rollback_locked_primary_accept() { prestop_watchdog_log "primary-write-commit-cleanup label=${label} reason=${reason} rollback_rc=${rollback_rc} release_rc=${release_rc} fail_closed=false" return 3 } +rollback_ambiguous_primary_publish() { + local label="${1:-fenced-primary-accept}" + local reason="${2:-publication-commit-rejected}" + local expected_token="${3:-}" + local rollback_rc=0 + local guard_restore_rc=0 + local publication_release_rc=0 + local commit_release_rc=0 + + if recover_committed_primary_write_publication "${expected_token}" "${label}"; then + return 0 + fi + + # The syncer live handler owns this shared data-volume lock from marker/DB + # validation through final DCS CAS and guard removal. On an ambiguous + # transport result, wait for that handler to finish (or win the lock before + # it starts), then re-guard and roll back while still owning the same lock. + # This prevents a late handler from deleting the caller's recovered guard. + if ! acquire_primary_write_publication_lock_for_rollback; then + if recover_committed_primary_write_publication "${expected_token}" "${label}"; then + return 0 + fi + # The bounded wait exceeds the live handler's operation deadline. If no + # exact durable receipt exists, the only safe terminal is a restored exact + # guard plus the strongest writer fence; a mismatched/stale receipt must + # never authorize committed recovery. + persist_primary_write_accept_guard "${expected_token}" || guard_restore_rc=1 + rollback_locked_primary_accept "${label}" "${reason}-publication-lock-timeout" || rollback_rc=$? + if [ "$(cat "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" 2>/dev/null)" != "${expected_token}" ]; then + guard_restore_rc=2 + fi + prestop_watchdog_log "primary-write-publish label=${label} rc=3 reason=publication-lock-timeout rollback_rc=${rollback_rc} guard_restore_rc=${guard_restore_rc} fail_closed=false" + return 3 + fi + if recover_committed_primary_write_publication "${expected_token}" "${label}"; then + return 0 + fi + if ! persist_primary_write_accept_guard "${expected_token}"; then + rollback_rc=1 + fi + rollback_fenced_primary_accept "${label}" "${reason}" || rollback_rc=1 + release_primary_write_publication_lock || publication_release_rc=1 + if ! release_primary_write_commit_lock; then + commit_release_rc=1 + force_release_primary_write_commit_lock || commit_release_rc=2 + fi + if [ "${rollback_rc}" -eq 0 ] && \ + [ "${publication_release_rc}" -eq 0 ] && \ + [ "${commit_release_rc}" -eq 0 ]; then + return 2 + fi + prestop_watchdog_log "primary-write-publish label=${label} rc=3 reason=${reason} rollback_rc=${rollback_rc} publication_release_rc=${publication_release_rc} commit_release_rc=${commit_release_rc} fail_closed=false" + return 3 +} set_primary_read_write() { local label="${1:-primary-read-write}" local role_guard="${2:-no-dcs-check}" - local accept_rc + local accept_rc accept_token publish_recovery_rc # Do not let any user-facing writer open before all required gates pass. # LOCAL is the user-facing root and remains fenced here; INTERNAL_LOCAL is @@ -588,12 +786,33 @@ set_primary_read_write() { return $? fi + # A DCS-guarded accept owns durable HA authority state across addon and + # syncer process lifetimes. Ordinary RunCycle promotion must see this marker + # and remain fail-closed until this exact caller receives the terminal + # primary-write-commit receipt and stages both ready markers, then syncer + # removes it inside the RunCycle HA mutex as the final publication step. + # Rollback deliberately leaves or recreates it. + if [ "${role_guard}" = "require-dcs-primary" ]; then + # Each accept owns a new exact transaction token. Clear only the previous + # completed receipt while the caller commit lock is held, then atomically + # publish the new durable guard before opening any writer plane. + if ! rm -f "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" || \ + [ -e "${PRIMARY_WRITE_PUBLICATION_COMMITTED_FILE}" ]; then + rollback_locked_primary_accept "${label}" "stale-publication-receipt-clear" + return $? + fi + accept_token="${POD_NAME:-unknown}:$$:$(date +%s):${RANDOM:-0}" + if ! persist_primary_write_accept_guard "${accept_token}"; then + rollback_locked_primary_accept "${label}" "accept-pending-publish" + return $? + fi + fi + # Unlock both user-root account planes while global read_only=ON still # rejects every user writer. The authoritative syncer operation is the - # final visible commit: under the HA mutex it refreshes DCS, renews the lease - # by resourceVersion CAS, opens/readbacks global writes, and publishes both - # ready markers before its HTTP response. Nothing below that response may - # unlock or publish, otherwise the next RunCycle could demote/release first. + # first authoritative phase: under the HA mutex it refreshes DCS, renews the + # lease by resourceVersion CAS, and opens/readbacks global writes. Marker + # staging and a second mutex-linearized publication phase follow below. if ! unlock_local_root_writes "${label}"; then rollback_locked_primary_accept "${label}" "local-root-unlock" return $? @@ -611,6 +830,31 @@ set_primary_read_write() { rollback_locked_primary_accept "${label}" "authority-commit-rejected" return $? fi + # Syncer intentionally leaves both addon-facing markers unpublished. Its + # exact success receipt means the handler finished every DCS/database + # mutation and delegated publication to this caller. If timeout releases + # us while the handler is still alive, this branch is not entered and the + # rollback below cannot be crossed by a late server marker. + if ! touch "${DATA_DIR}/.primary-read-write-ready" || \ + ! mark_replication_ready; then + rollback_locked_primary_accept "${label}" "ready-publish" + return $? + fi + # Marker staging is not publication: the durable guard keeps roleProbe + # closed. Commit visibility with a second bounded syncer operation that + # owns the same HA mutex as RunCycle across fresh DCS authority/lease CAS, + # exact marker+DB validation, and guard removal. Any error (including an + # ambiguous transport timeout after server-side removal) enters a shared + # publication lock before recreating the guard and rolling back. The live + # handler owns the same lock from validation through actual guard removal, + # so caller recovery cannot race a stale server snapshot. + if ! authoritative_primary_write_publish "${label}" "${accept_token}"; then + publish_recovery_rc=0 + rollback_ambiguous_primary_publish "${label}" "publication-commit-rejected" "${accept_token}" || publish_recovery_rc=$? + if [ "${publish_recovery_rc}" -ne 0 ]; then + return "${publish_recovery_rc}" + fi + fi else if ! "${INTERNAL_LOCAL[@]}" -e "SET GLOBAL read_only = 0;" 2>/dev/null && \ ! "${INTERNAL_LOCAL[@]}" -e "SET GLOBAL read_only = 'OFF';" 2>/dev/null; then @@ -629,7 +873,7 @@ set_primary_read_write() { prestop_watchdog_log "primary-write-commit-lock label=${label} rc=3 reason=release-failed rollback_rc=${accept_rc:-0} fail_closed=false" return 3 fi - prestop_watchdog_log "primary-write-accept label=${label} rc=0 required-gates=passed authority-commit-acknowledged=true" + prestop_watchdog_log "primary-write-accept label=${label} rc=0 required-gates=passed authority-commit-acknowledged=true publication-commit=ha-mutex" return 0 } rollback_fenced_primary_accept() { @@ -680,9 +924,9 @@ mark_replication_pending() { touch ${DATA_DIR}/.replication-pending } mark_replication_ready() { - touch ${DATA_DIR}/.replication-ready - touch ${DATA_DIR}/.sql-listener-ready - rm -f ${DATA_DIR}/.replication-pending ${DATA_DIR}/.replication-divergence-pending + touch "${DATA_DIR}/.replication-ready" || return 1 + touch "${DATA_DIR}/.sql-listener-ready" || return 1 + rm -f "${DATA_DIR}/.replication-pending" "${DATA_DIR}/.replication-divergence-pending" || return 1 } mark_replication_divergence_pending() { mark_replication_pending @@ -704,12 +948,21 @@ accept_syncer_primary_promotion_from_replica_path() { prestop_watchdog_log "replica-path-accept-dcs-primary label=${label} action=accept-primary-promotion" # r9 first-red: replica fencing can begin while syncer still reports - # secondary, then race with syncer promoting this same Pod. Keep the - # global read_only fence ON until the existing full-primary acceptance - # takes control. MariaDB read_only is server-global, so turning it OFF here - # would also expose ordinary business users, not just kb_internal_root. + # secondary, then race with syncer promoting this same Pod. Promotion can + # win before set_replica_read_only reaches its first lock operation, so an + # rc=2 from that path does not itself prove that global read_only is ON. + # Positively establish the fail-closed fence before the existing full- + # primary acceptance takes control. MariaDB read_only is server-global, so + # turning it OFF here would also expose ordinary business users, not just + # kb_internal_root. # Syncer's DCS-authoritative local leader heartbeat must instead use its # dedicated READ_ONLY ADMIN connection while suppressing binlog output. + if ! read_only_is_fail_closed; then + if ! set_fail_closed_read_only "${label}-pre-accept"; then + prestop_watchdog_log "replica-path-primary-accept-rollback label=${label} rc=1 reason=pre-accept-fence-failed fail_closed=false" + return 3 + fi + fi # Re-check DCS immediately before the full acceptance to close the role # decision window without opening a write window first. role="$(query_local_syncer_role || true)" @@ -1876,7 +2129,7 @@ start_mariadbd_process() { # 11.4 AND 11.8), turning fail-closed intent into fail-open on every shipped # version. The portable boolean ON is used here; set_fail_closed_read_only # still upgrades to NO_LOCK_NO_ADMIN post-start where the engine supports it. - docker-entrypoint.sh mariadbd \ + /scripts/mariadb-image-entrypoint.sh run "${DATA_DIR}" mariadbd \ --defaults-extra-file=${DATA_DIR}/runtime-overrides.cnf \ --server-id=${SERVICE_ID} \ --gtid-domain-id=${SERVICE_ID} \ @@ -2086,6 +2339,7 @@ query_primary_service_server_id() { } local_primary_role_published() { [ ! -f "${DATA_DIR}/master.info" ] && \ + [ ! -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] && \ [ -f "${DATA_DIR}/.primary-read-write-ready" ] && \ [ -f "${DATA_DIR}/.sql-listener-ready" ] && \ mariadbd_listen_on_all_interfaces @@ -2110,8 +2364,33 @@ local_has_user_tables() { [ "${table_count}" -gt 0 ] } reconcile_sql_listener_for_syncer_primary_once() { - local now primary_sid role primary_ready remote_root_fence master_info listener_wildcard + local now primary_sid role primary_ready remote_root_fence master_info listener_wildcard retry_rc [ ! -f "${DATA_DIR}/.prestop-fence-started" ] || return 0 + # A durable accept guard means an earlier exact primary-write transaction + # failed closed before publication. While that guard exists, getrole must + # continue to report the database truth (secondary/read-only), so waiting + # for a published `primary` here creates a permanent self-denying state. + # Retry only through the same DCS/lease-CAS commit path that created the + # transaction: a non-authoritative member is rejected and rolled back with + # the guard preserved, while the member whose HA loop retains the deferred + # lease can finish commit+publication. Never remove the guard here. + if [ -f "${PRIMARY_WRITE_ACCEPT_PENDING_FILE}" ] && \ + [ ! -f "${DATA_DIR}/master.info" ]; then + prestop_watchdog_log "runtime-primary-accept-retry-begin reason=durable-accept-pending authority=syncer-lease-cas" + if mariadbd_listen_on_all_interfaces; then + expose_sql_listener_for_primary_role "syncer-primary-accept-pending-retry" "fenced-promotion" + retry_rc=$? + else + set_primary_read_write "syncer-primary-accept-pending-retry" "require-dcs-primary" + retry_rc=$? + fi + if [ "${retry_rc}" -eq 0 ]; then + prestop_watchdog_log "runtime-primary-accept-retry-complete authority=syncer-lease-cas" + return 0 + fi + prestop_watchdog_log "runtime-primary-accept-retry-defer rc=${retry_rc} guard=preserved" + return "${retry_rc}" + fi # alpha.80 v1 (Helen): the alpha.76 `switchover_fence_active_is_fresh` # early-skip has been removed. alpha.79 v1 minimalist deleted the # marker writer in switchover.sh, so this check could never observe @@ -2282,6 +2561,22 @@ reconcile_sql_listener_for_syncer_secondary_once() { [ ! -f "${DATA_DIR}/.prestop-fence-started" ] || return 0 role="$(query_local_syncer_role || true)" [ "${role}" = "secondary" ] || return 0 + # Fresh pod-0 is intentionally published locally before the DCS leader + # ConfigMap exists. During that short bootstrap window syncerctl must report + # secondary: primary publication is DCS-authoritative. Do not feed that + # pre-DCS observation back into the database by fencing the only writable + # bootstrap member, or InitializeDCS can no longer discover a leader and the + # cluster deadlocks with every member secondary. + # + # This defer is narrow and fail-closed on uncertainty: it requires the full + # local-primary marker/listener contract plus an explicit read_only=OFF + # result. During a real switchover syncer demotes the former primary first; + # once read_only is ON this guard no longer applies and normal secondary + # fencing/follow proceeds. + if local_primary_role_published && read_only_is_writable; then + prestop_watchdog_log "runtime-secondary-listener-reconcile-defer role=${role} reason=pre-dcs-local-primary-writable" + return 0 + fi set_replica_read_only "runtime-secondary-reconcile" slave_rejoin_rc=$? if [ "${slave_rejoin_rc}" -eq 2 ]; then diff --git a/addons/mariadb/scripts/replication-roleprobe.sh b/addons/mariadb/scripts/replication-roleprobe.sh index f10441e029..ae82f2f7e7 100644 --- a/addons/mariadb/scripts/replication-roleprobe.sh +++ b/addons/mariadb/scripts/replication-roleprobe.sh @@ -47,6 +47,10 @@ pending_file() { printf "%s/.replication-pending" "$(data_dir)" } +primary_write_accept_pending_file() { + printf "%s/.primary-write-accept-pending" "$(data_dir)" +} + remote_root_fence_file() { printf "%s/.remote-root-fence-role" "$(data_dir)" } @@ -765,6 +769,7 @@ check_role() { apply_remote_root_fence "secondary" || { not_ready; return $?; } printf '%s' "secondary" else + [ ! -f "$(primary_write_accept_pending_file)" ] || { not_ready; return $?; } primary_listener_ready || { not_ready; return $?; } primary_read_write_ready || { not_ready; return $?; } apply_remote_root_fence "primary" || { not_ready; return $?; } diff --git a/addons/mariadb/templates/configmap-scripts-replication.yaml b/addons/mariadb/templates/configmap-scripts-replication.yaml index 6443471b54..78bfdbc640 100644 --- a/addons/mariadb/templates/configmap-scripts-replication.yaml +++ b/addons/mariadb/templates/configmap-scripts-replication.yaml @@ -54,6 +54,8 @@ data: {{- .Files.Get "scripts/seed-replication-mode-overrides.sh" | nindent 4 }} reconcile-runtime-overrides.sh: |- {{- .Files.Get "scripts/reconcile-runtime-overrides.sh" | nindent 4 }} + mariadb-image-entrypoint.sh: |- + {{- .Files.Get "scripts/mariadb-image-entrypoint.sh" | nindent 4 }} replication-entrypoint.sh: |- {{- .Files.Get "scripts/replication-entrypoint.sh" | nindent 4 }} replication-prestop.sh: |-