From b8da48add7dfc6f6c28f85b21c035907ca9cb347 Mon Sep 17 00:00:00 2001 From: wei Date: Sat, 25 Jul 2026 16:54:54 +0800 Subject: [PATCH 1/4] fix(falkordb): bound standalone switchover action --- .../falkordb_switchover_spec.sh | 260 +++++++++++++++- .../switchover_timeout_contract_spec.sh | 31 ++ .../falkordb/scripts/falkordb-switchover.sh | 278 +++++++++++++++--- addons/falkordb/templates/cmpd-falkordb.yaml | 3 + 4 files changed, 536 insertions(+), 36 deletions(-) create mode 100644 addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh diff --git a/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh b/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh index cc6ca42620..9787a96316 100644 --- a/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh +++ b/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh @@ -117,6 +117,35 @@ master_host:redis-master" End End + Context "run_redis_cli()" + It "enforces a command timeout outside unit-test mode" + ut_mode="false" + timeout() { + echo "TIMEOUT:$*" + } + When call run_redis_cli -h redis1 PING + The status should be success + The output should include "TIMEOUT:-k 2 5 redis-cli -h redis1 PING" + End + + It "actually terminates a hanging redis-cli process" + run_hanging_redis_cli_case() { + local stub_dir + stub_dir=$(mktemp -d) + printf '%s\n' '#!/bin/bash' 'sleep 60' > "$stub_dir/redis-cli" + chmod +x "$stub_dir/redis-cli" + PATH="$stub_dir:$PATH" + ut_mode="false" + run_redis_cli -h redis1 PING + local status=$? + rm -rf "$stub_dir" + return "$status" + } + When call run_hanging_redis_cli_case + The status should be failure + End + End + Context "check_redis_kernel_status()" setup() { export REDIS_POD_FQDN_LIST="redis1,redis2,redis3" @@ -280,9 +309,13 @@ master_host:redis-master" } check_redis_kernel_status() { return 0; } set_redis_priorities() { return 0; } + wait_sentinel_sees_priority_bias() { return 0; } execute_sentinel_failover() { return 0; } check_switchover_result() { return 0; } - recover_redis_priorities() { return 0; } + recover_redis_priorities() { + echo "All FalkorDB config set replica-priority recovered." + return 0 + } When call switchover_with_candidate The status should be success @@ -290,6 +323,229 @@ master_host:redis-master" The stderr should equal "" End + It "should restore priorities when Sentinel cache confirmation fails" + load_common_library() { return 0; } + check_environment_exist() { return 0; } + export KB_SWITCHOVER_CANDIDATE_FQDN="redis2" + check_redis_role() { echo "secondary"; } + check_redis_kernel_status() { echo "redis1"; } + set_redis_priorities() { + priorities_mutated=true + return 0 + } + wait_sentinel_sees_priority_bias() { return 1; } + recover_redis_priorities() { + echo "RESTORED" + priorities_mutated=false + return 0 + } + When call run_switchover_action + The status should be failure + The stdout should include "RESTORED" + End + + It "should restore priorities when Sentinel failover fails" + load_common_library() { return 0; } + check_environment_exist() { return 0; } + export KB_SWITCHOVER_CANDIDATE_FQDN="redis2" + check_redis_role() { echo "secondary"; } + check_redis_kernel_status() { echo "redis1"; } + set_redis_priorities() { + priorities_mutated=true + return 0 + } + wait_sentinel_sees_priority_bias() { return 0; } + execute_sentinel_failover() { return 1; } + recover_redis_priorities() { + echo "RESTORED" + priorities_mutated=false + return 0 + } + When call run_switchover_action + The status should be failure + The stdout should include "RESTORED" + End + + It "should wait for all Sentinel caches to observe the target bias" + export SENTINEL_POD_FQDN_LIST="sentinel1,sentinel2" + export REDIS_POD_FQDN_LIST="redis1,redis2,redis3" + ORIGINAL_PRIORITIES["redis1"]=100 + ORIGINAL_PRIORITIES["redis2"]=100 + ORIGINAL_PRIORITIES["redis3"]=0 + sentinel_observed_replica_priority() { + case "$1:$2" in + sentinel1:redis2|sentinel2:redis2) echo "1" ;; + sentinel1:redis3|sentinel2:redis3) echo "0" ;; + esac + } + When call wait_sentinel_sees_priority_bias "redis2" "redis1" + The status should be success + The stdout should include "All Sentinel replica priority caches confirmed" + End + + It "should parse the target priority from Sentinel replica output" + export REDIS_CLI_TLS_CMD="" + export CUSTOM_SENTINEL_MASTER_NAME="redis" + export SENTINEL_SERVICE_PORT="26379" + export SENTINEL_PASSWORD="" + redis-cli() { + cat <<'EOF' +1) "name" +2) "not-redis2.redis-headless.default.svc.cluster.local:6379" +3) "slave-priority" +4) "100" +5) "name" +6) "redis2.redis-headless.default.svc.cluster.local:6379" +7) "ip" +8) "redis2.redis-headless.default.svc.cluster.local" +9) "slave-priority" +10) "1" +11) "name" +12) "redis3.redis-headless.default.svc.cluster.local:6379" +13) "slave-priority" +14) "100" +EOF + } + When call sentinel_observed_replica_priority "sentinel1" "redis2.redis-headless.default.svc.cluster.local" + The status should be success + The output should equal "1" + End + + It "should not accept a different FQDN with the same pod label" + When call same_fqdn \ + "redis2.other-headless.other.svc.cluster.local" \ + "redis2.redis-headless.default.svc.cluster.local" + The status should be failure + End + + It "should parse an exact target after a same-label collision" + export REDIS_CLI_TLS_CMD="" + export CUSTOM_SENTINEL_MASTER_NAME="redis" + export SENTINEL_SERVICE_PORT="26379" + export SENTINEL_PASSWORD="" + redis-cli() { + cat <<'EOF' +1) "name" +2) "redis2.other-headless.other.svc.cluster.local:6379" +3) "slave-priority" +4) "100" +5) "name" +6) "redis2.redis-headless.default.svc.cluster.local:6379" +7) "slave-priority" +8) "1" +EOF + } + When call sentinel_observed_replica_priority "sentinel1" "redis2.redis-headless.default.svc.cluster.local" + The status should be success + The output should equal "1" + End + + It "should reject a requested candidate absent from the exact pod list" + export REDIS_POD_FQDN_LIST="redis1.redis-headless.default.svc.cluster.local,redis2.redis-headless.default.svc.cluster.local" + export KB_SWITCHOVER_CANDIDATE_FQDN="redis2.other-headless.other.svc.cluster.local" + check_redis_role() { + echo "secondary" + } + When call switchover_with_candidate + The status should be failure + The stderr should include "not an exact member of REDIS_POD_FQDN_LIST" + End + + It "should fail a successful action when priority cleanup fails" + load_common_library() { return 0; } + check_environment_exist() { return 0; } + export KB_SWITCHOVER_CANDIDATE_FQDN="redis2" + switchover_with_candidate() { + priorities_mutated=true + return 0 + } + recover_redis_priorities() { return 1; } + When call run_switchover_action + The status should be failure + The stderr should include "Failed to restore" + End + + It "should keep priority recovery retries within the supervisor grace" + ORIGINAL_PRIORITIES=() + ORIGINAL_PRIORITIES["redis1"]=100 + ORIGINAL_PRIORITIES["redis2"]=100 + ORIGINAL_PRIORITIES["redis3"]=0 + call_func_with_retry() { + echo "RETRY_CONTRACT:$1:$2:$3:$4" + return 0 + } + When call recover_redis_priorities + The status should be success + The output should include "RETRY_CONTRACT:2:1:execute_sub_command:redis1" + The output should include "RETRY_CONTRACT:2:1:execute_sub_command:redis2" + The output should include "RETRY_CONTRACT:2:1:execute_sub_command:redis3" + End + + It "should complete real parallel timeout retries within the cleanup grace" + run_real_priority_recovery_budget_case() { + local stub_dir + local started + local elapsed + stub_dir=$(mktemp -d) + printf '%s\n' '#!/bin/bash' 'sleep 60' > "$stub_dir/redis-cli" + chmod +x "$stub_dir/redis-cli" + PATH="$stub_dir:$PATH" + ut_mode="false" + ORIGINAL_PRIORITIES=() + ORIGINAL_PRIORITIES["redis1"]=100 + ORIGINAL_PRIORITIES["redis2"]=100 + ORIGINAL_PRIORITIES["redis3"]=0 + started=$SECONDS + recover_redis_priorities + local status=$? + elapsed=$((SECONDS - started)) + rm -rf "$stub_dir" + echo "RECOVERY_ELAPSED:$elapsed" + [[ $status -ne 0 && $elapsed -ge 10 && $elapsed -lt 20 ]] + } + When call run_real_priority_recovery_budget_case + The status should be success + The output should include "RECOVERY_ELAPSED:" + The stderr should include "failed after 2 retries." + End + + It "should route the production main through the bounded supervisor" + supervise_switchover_action() { + echo "PRODUCTION_SUPERVISOR:$*" + return 23 + } + When call falkordb_switchover_main payload + The status should be failure + The output should include \ + "PRODUCTION_SUPERVISOR:420 60 /bin/bash" + The output should include \ + "--falkordb-switchover-deadline-child payload" + End + + It "should run TERM cleanup when a supervised child hangs" + When run supervise_switchover_action 1 2 /bin/bash -c \ + 'trap "echo TERM_CLEANUP; exit 1" TERM; while :; do sleep 1; done' + The status should be failure + The output should include "TERM_CLEANUP" + The stderr should include "Terminated" + End + + It "should force-kill a supervised child that ignores TERM" + run_forced_kill_case() { + ( + supervise_switchover_action 1 1 /bin/bash -c \ + 'trap "" TERM; while :; do sleep 1; done' + ) 2>/dev/null + local status=$? + echo "FORCED_KILL_STATUS:$status" + return "$status" + } + When call run_forced_kill_case + The status should be failure + The output should include "FORCED_KILL_STATUS:" + The stderr should include "Killed" + End + It "should fail when candidate is primary" check_redis_role() { echo "primary" @@ -369,4 +625,4 @@ master_host:redis-master" End End End -End \ No newline at end of file +End diff --git a/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh b/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh new file mode 100644 index 0000000000..9e74b5f587 --- /dev/null +++ b/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh @@ -0,0 +1,31 @@ +#!/bin/bash + +Describe "FalkorDB switchover timeout contract" + chart_path() { + printf '%s/addons/falkordb\n' "$(cd "${SHELLSPEC_PROJECT_ROOT:-.}" && pwd)" + } + + rendered_switchover_timeout() { + helm template test "$(chart_path)" \ + --show-only templates/cmpd-falkordb.yaml \ + | ruby -ryaml -e ' + component = YAML.load_stream($stdin.read).compact.find do |doc| + doc["kind"] == "ComponentDefinition" && + doc.dig("metadata", "name") == "falkordb-4-1.2.0-alpha.0" + end + abort "rendered ComponentDefinition not found" unless component + timeout = component.dig( + "spec", "lifecycleActions", "switchover", "timeoutSeconds" + ) + abort "rendered switchover timeout is #{timeout.inspect}" unless timeout == -1 + puts timeout + ' + } + + It "renders the negative kbagent timeout on the intended lifecycle action" + When call rendered_switchover_timeout + The status should be success + The output should equal "-1" + End + +End diff --git a/addons/falkordb/scripts/falkordb-switchover.sh b/addons/falkordb/scripts/falkordb-switchover.sh index bf6d543aac..778434e2d3 100644 --- a/addons/falkordb/scripts/falkordb-switchover.sh +++ b/addons/falkordb/scripts/falkordb-switchover.sh @@ -22,6 +22,46 @@ test || __() { declare -A ORIGINAL_PRIORITIES redis_service_port=${SERVICE_PORT:-6379} +readonly redis_cli_timeout_seconds=5 +readonly switchover_action_timeout_seconds=420 +readonly switchover_cleanup_grace_seconds=60 +readonly sentinel_priority_wait_seconds=30 +readonly priority_recovery_attempts=2 +readonly priority_recovery_retry_interval_seconds=1 +priorities_mutated=false + +normalize_fqdn() { + local fqdn="${1%.}" + printf '%s\n' "${fqdn,,}" +} + +same_fqdn() { + [[ "$(normalize_fqdn "$1")" == "$(normalize_fqdn "$2")" ]] +} + +fqdn_in_csv() { + local target_fqdn="$1" + local csv="$2" + local -a fqdns + local fqdn + IFS=',' read -ra fqdns <<< "$csv" + for fqdn in "${fqdns[@]}"; do + same_fqdn "$fqdn" "$target_fqdn" && return 0 + done + return 1 +} + +run_redis_cli() { + local -a tls_args=() + if [[ -n "${REDIS_CLI_TLS_CMD:-}" ]]; then + read -ra tls_args <<< "$REDIS_CLI_TLS_CMD" + fi + if [[ "$ut_mode" == "true" ]]; then + redis-cli "${tls_args[@]}" "$@" + else + timeout -k 2 "$redis_cli_timeout_seconds" redis-cli "${tls_args[@]}" "$@" + fi +} load_common_library() { # the common.sh scripts is mounted to the same path which is defined in the cmpd.spec.scripts @@ -60,9 +100,9 @@ check_redis_role() { unset_xtrace_when_ut_mode_false local role_info if [[ -z "$REDIS_DEFAULT_PASSWORD" ]]; then - role_info=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" info replication) + role_info=$(run_redis_cli -h "$host" -p "$port" info replication) else - role_info=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" -a "$REDIS_DEFAULT_PASSWORD" info replication) + role_info=$(run_redis_cli -h "$host" -p "$port" -a "$REDIS_DEFAULT_PASSWORD" info replication) fi status=$? set_xtrace_when_ut_mode_false @@ -112,14 +152,14 @@ check_switchover_result() { local initial_master="$2" local max_wait=300 local wait_interval=5 - local elapsed=0 + local deadline=$((SECONDS + max_wait)) - while [[ $elapsed -lt $max_wait ]]; do + while [[ $SECONDS -lt $deadline ]]; do local current_master if current_master=$(check_redis_kernel_status); then # if expected_master is specified, check if it is achieved if ! is_empty "$expected_master"; then - if [[ "$current_master" = "$expected_master"* ]]; then + if same_fqdn "$current_master" "$expected_master"; then echo "Switchover successful: $expected_master is now master" return 0 fi @@ -135,7 +175,6 @@ check_switchover_result() { fi fi sleep_when_ut_mode_false $wait_interval - elapsed=$((elapsed + wait_interval)) done if ! is_empty "$expected_master"; then @@ -154,9 +193,9 @@ check_connectivity() { local result unset_xtrace_when_ut_mode_false if ! is_empty "$password"; then - result=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" -a "$password" PING) + result=$(run_redis_cli -h "$host" -p "$port" -a "$password" PING) else - result=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" PING) + result=$(run_redis_cli -h "$host" -p "$port" PING) fi set_xtrace_when_ut_mode_false if [[ "$result" == "PONG" ]]; then @@ -173,13 +212,15 @@ execute_sub_command() { local port=$2 local password=$3 local command=$4 + local -a command_args + read -ra command_args <<< "$command" local output unset_xtrace_when_ut_mode_false if ! is_empty "$password"; then - output=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" -a "$password" $command) + output=$(run_redis_cli -h "$host" -p "$port" -a "$password" "${command_args[@]}") else - output=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" $command) + output=$(run_redis_cli -h "$host" -p "$port" "${command_args[@]}") fi local status=$? set_xtrace_when_ut_mode_false @@ -198,13 +239,15 @@ redis_config_get() { local port=$2 local password=$3 local command=$4 + local -a command_args + read -ra command_args <<< "$command" local output unset_xtrace_when_ut_mode_false if ! is_empty "$password"; then - output=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" -a "$password" $command) + output=$(run_redis_cli -h "$host" -p "$port" -a "$password" "${command_args[@]}") else - output=$(redis-cli $REDIS_CLI_TLS_CMD -h "$host" -p "$port" $command) + output=$(run_redis_cli -h "$host" -p "$port" "${command_args[@]}") fi local status=$? set_xtrace_when_ut_mode_false @@ -250,6 +293,84 @@ execute_sentinel_failover() { return 0 } +sentinel_observed_replica_priority() { + local sentinel_fqdn="$1" + local replica_fqdn="$2" + local master_name="${CUSTOM_SENTINEL_MASTER_NAME:-$REDIS_COMPONENT_NAME}" + local replica_identity + replica_identity=$(normalize_fqdn "$replica_fqdn") + local output + + unset_xtrace_when_ut_mode_false + if [[ -z "$SENTINEL_PASSWORD" ]]; then + output=$(run_redis_cli -h "$sentinel_fqdn" -p "$SENTINEL_SERVICE_PORT" \ + SENTINEL REPLICAS "$master_name") + else + output=$(run_redis_cli -h "$sentinel_fqdn" -p "$SENTINEL_SERVICE_PORT" \ + -a "$SENTINEL_PASSWORD" SENTINEL REPLICAS "$master_name") + fi + local status=$? + set_xtrace_when_ut_mode_false + [[ $status -eq 0 ]] || return 1 + + printf '%s\n' "$output" \ + | tr -d '"' \ + | sed 's/.*) //' \ + | awk -v candidate="$replica_identity" ' + previous == "name" { + identity = tolower($0) + sub(/:[0-9]+$/, "", identity) + sub(/\.$/, "", identity) + in_candidate = (identity == candidate) + } + in_candidate && previous == "slave-priority" { print; exit } + { previous = $0 } + ' +} + +wait_sentinel_sees_priority_bias() { + local candidate_fqdn="$1" + local current_master="$2" + local deadline=$((SECONDS + sentinel_priority_wait_seconds)) + local -a sentinel_pod_fqdn_list redis_pod_fqdn_list + + while [[ $SECONDS -lt $deadline ]]; do + IFS=',' read -ra sentinel_pod_fqdn_list <<< "${SENTINEL_POD_FQDN_LIST}" + IFS=',' read -ra redis_pod_fqdn_list <<< "${REDIS_POD_FQDN_LIST}" + local total=0 + local confirmed=0 + local sentinel_pod_fqdn redis_pod_fqdn + + for sentinel_pod_fqdn in "${sentinel_pod_fqdn_list[@]}"; do + for redis_pod_fqdn in "${redis_pod_fqdn_list[@]}"; do + local expected_priority=100 + local observed_priority + + # Sentinel does not list the current primary as a replica before failover. + same_fqdn "$redis_pod_fqdn" "$current_master" && continue + same_fqdn "$redis_pod_fqdn" "$candidate_fqdn" && expected_priority=1 + [[ "${ORIGINAL_PRIORITIES[$redis_pod_fqdn]}" == "0" ]] \ + && ! same_fqdn "$redis_pod_fqdn" "$candidate_fqdn" \ + && expected_priority=0 + + total=$((total + 1)) + observed_priority=$(sentinel_observed_replica_priority \ + "$sentinel_pod_fqdn" "$redis_pod_fqdn") || true + [[ "$observed_priority" == "$expected_priority" ]] && confirmed=$((confirmed + 1)) + done + done + + if [[ $total -gt 0 && $confirmed -eq $total ]]; then + echo "All Sentinel replica priority caches confirmed targeted bias for $candidate_fqdn." + return 0 + fi + sleep_when_ut_mode_false 1 + done + + echo "Error: Sentinel did not confirm targeted priority bias for $candidate_fqdn within ${sentinel_priority_wait_seconds}s" >&2 + return 1 +} + # set target candidate highest priority to make sure it will be promoted to master set_redis_priorities() { local candidate_fqdn="$1" @@ -261,24 +382,34 @@ set_redis_priorities() { # Get original priority local redis_get_cmd="CONFIG GET replica-priority" + local original_priority_output local original_priority - original_priority=$(redis_config_get "$redis_pod_fqdn" "$redis_service_port" "$REDIS_DEFAULT_PASSWORD" "$redis_get_cmd" | sed -n '2p') - status=$? - if [ $status -ne 0 ]; then + if ! original_priority_output=$(redis_config_get \ + "$redis_pod_fqdn" "$redis_service_port" "$REDIS_DEFAULT_PASSWORD" "$redis_get_cmd"); then echo "Error: Failed to get replica-priority for $redis_pod_fqdn" >&2 return 1 fi + original_priority=$(printf '%s\n' "$original_priority_output" | sed -n '2p') + if [[ ! "$original_priority" =~ ^[0-9]+$ ]]; then + echo "Error: Invalid replica-priority for $redis_pod_fqdn: $original_priority" >&2 + return 1 + fi # Save original priority to global variable ORIGINAL_PRIORITIES[$redis_pod_fqdn]=$original_priority local redis_set_cmd - if [[ "$redis_pod_fqdn" = "$candidate_fqdn"* ]]; then + if same_fqdn "$redis_pod_fqdn" "$candidate_fqdn"; then redis_set_cmd="CONFIG SET replica-priority 1" + elif [[ "$original_priority" == "0" ]]; then + echo "Preserving never-promote replica-priority=0 on $redis_pod_fqdn." + continue else redis_set_cmd="CONFIG SET replica-priority 100" fi + # The command can apply server-side before its response is lost or times out. + priorities_mutated=true call_func_with_retry 3 5 execute_sub_command "$redis_pod_fqdn" "$redis_service_port" "$REDIS_DEFAULT_PASSWORD" "$redis_set_cmd" || return 1 done return 0 @@ -286,19 +417,80 @@ set_redis_priorities() { # recover all redis replica-priority recover_redis_priorities() { - local -a redis_pod_fqdn_list - IFS=',' read -ra redis_pod_fqdn_list <<< "${REDIS_POD_FQDN_LIST}" - echo "Recovering all FalkorDB replica-priority..." - for redis_pod_fqdn in "${redis_pod_fqdn_list[@]}"; do + local redis_pod_fqdn + local failed=0 + local -a restore_pids=() + for redis_pod_fqdn in "${!ORIGINAL_PRIORITIES[@]}"; do local redis_set_recover_cmd="CONFIG SET replica-priority ${ORIGINAL_PRIORITIES[$redis_pod_fqdn]}" - call_func_with_retry 3 5 execute_sub_command "$redis_pod_fqdn" "$redis_service_port" "$REDIS_DEFAULT_PASSWORD" "$redis_set_recover_cmd" || return 1 + # Restore members in parallel so topology size does not multiply the TERM + # grace budget. Each worker is bounded by two (5s + 2s kill-grace) calls + # and one 1s retry sleep, for a worst-case worker budget below 15s. + ( + call_func_with_retry "$priority_recovery_attempts" \ + "$priority_recovery_retry_interval_seconds" execute_sub_command \ + "$redis_pod_fqdn" "$redis_service_port" "$REDIS_DEFAULT_PASSWORD" \ + "$redis_set_recover_cmd" + ) & + restore_pids+=("$!") done + local restore_pid + for restore_pid in "${restore_pids[@]}"; do + wait "$restore_pid" || failed=1 + done + [[ $failed -eq 0 ]] || return 1 + priorities_mutated=false echo "All FalkorDB config set replica-priority recovered." return 0 } +cleanup_redis_priorities() { + if [[ "$priorities_mutated" == "true" ]]; then + if ! recover_redis_priorities; then + echo "Error: Failed to restore one or more FalkorDB replica priorities" >&2 + return 1 + fi + fi + return 0 +} + +handle_termination() { + local signal="$1" + trap - EXIT TERM INT + cleanup_redis_priorities + echo "Error: FalkorDB switchover interrupted by $signal" >&2 + exit 1 +} + +supervise_switchover_action() { + local action_timeout="$1" + local cleanup_grace="$2" + shift 2 + exec timeout -k "$cleanup_grace" "$action_timeout" "$@" +} + +falkordb_switchover_main() { + if [[ "${1:-}" != "--falkordb-switchover-deadline-child" ]]; then + if ! command -v timeout >/dev/null 2>&1; then + echo "Error: timeout command is required for bounded FalkorDB switchover" >&2 + return 1 + fi + supervise_switchover_action \ + "$switchover_action_timeout_seconds" "$switchover_cleanup_grace_seconds" \ + /bin/bash "$0" --falkordb-switchover-deadline-child "$@" + return $? + fi + + shift + run_switchover_action "$@" +} + switchover_with_candidate() { + if ! fqdn_in_csv "$KB_SWITCHOVER_CANDIDATE_FQDN" "$REDIS_POD_FQDN_LIST"; then + echo "Error: Candidate node $KB_SWITCHOVER_CANDIDATE_FQDN is not an exact member of REDIS_POD_FQDN_LIST" >&2 + return 1 + fi + # check the role of candidate before switchover local candidate_role candidate_role=$(check_redis_role "$KB_SWITCHOVER_CANDIDATE_FQDN" "$redis_service_port") @@ -311,14 +503,14 @@ switchover_with_candidate() { local initial_master initial_master=$(check_redis_kernel_status) || return 1 - local redis_get_cmd="CONFIG GET replica-priority" - local redis_set_switchover_cmd="CONFIG SET replica-priority 1" - local redis_set_lowest_priority_cmd="CONFIG SET replica-priority 100" - # set target candidate highest priority to make sure it will be promoted to master unset_xtrace_when_ut_mode_false set_redis_priorities "$KB_SWITCHOVER_CANDIDATE_FQDN" || return 1 + # Sentinel caches replica priority independently on each pod. Wait until all + # Sentinels see the requested bias before asking any one of them to fail over. + wait_sentinel_sees_priority_bias "$KB_SWITCHOVER_CANDIDATE_FQDN" "$initial_master" || return 1 + # do switchover execute_sentinel_failover "$CUSTOM_SENTINEL_MASTER_NAME" || return 1 @@ -330,7 +522,6 @@ switchover_with_candidate() { recover_redis_priorities || return 1 set_xtrace_when_ut_mode_false - echo "All FalkorDB config set replica-priority recovered." } switchover_without_candidate() { @@ -346,6 +537,28 @@ switchover_without_candidate() { # check_switchover_result "" "$initial_master" || return 1 } +run_switchover_action() { + load_common_library || return 1 + check_environment_exist || return 1 + trap cleanup_redis_priorities EXIT + trap 'handle_termination TERM' TERM + trap 'handle_termination INT' INT + + local action_status=0 + if is_empty "$KB_SWITCHOVER_CANDIDATE_FQDN"; then + switchover_without_candidate || action_status=$? + else + switchover_with_candidate || action_status=$? + fi + + local cleanup_status=0 + cleanup_redis_priorities || cleanup_status=$? + trap - EXIT TERM INT + + [[ $action_status -eq 0 ]] || return "$action_status" + return "$cleanup_status" +} + # This is magic for shellspec ut framework. # Sometime, functions are defined in a single shell script. # You will want to test it. but you do not want to run the script. @@ -353,11 +566,8 @@ switchover_without_candidate() { # end here. The script path is assigned to the __SOURCED__ variable. ${__SOURCED__:+false} : || return 0 -# main -load_common_library -check_environment_exist || exit 1 -if is_empty "$KB_SWITCHOVER_CANDIDATE_FQDN"; then - switchover_without_candidate || exit 1 -else - switchover_with_candidate || exit 1 -fi +# kbagent's positive action timeout is capped at 60s, while Sentinel convergence +# can legitimately exceed that. The child process therefore owns the lifecycle, +# but is still supervised by a hard wall-clock deadline. The kill grace lets its +# TERM/EXIT cleanup restore any temporary replica-priority mutations. +falkordb_switchover_main "$@" diff --git a/addons/falkordb/templates/cmpd-falkordb.yaml b/addons/falkordb/templates/cmpd-falkordb.yaml index fcbc4bf70a..a9966d2369 100644 --- a/addons/falkordb/templates/cmpd-falkordb.yaml +++ b/addons/falkordb/templates/cmpd-falkordb.yaml @@ -361,6 +361,9 @@ spec: args: - /scripts/redis-account-provision.sh switchover: + # The script enforces per-command and whole-action wall-clock deadlines. + # Disable kbagent's 30s default so Sentinel convergence can finish first. + timeoutSeconds: -1 exec: container: falkordb command: From 6a2d6c7dfc4d7353258565c775387ef22ac937ce Mon Sep 17 00:00:00 2001 From: wei Date: Fri, 31 Jul 2026 11:50:16 +0800 Subject: [PATCH 2/4] fix(falkordb): match Sentinel announce addresses --- .../falkordb_switchover_spec.sh | 125 +++++++++++--- .../falkordb/scripts/falkordb-switchover.sh | 162 ++++++++++++++++-- 2 files changed, 247 insertions(+), 40 deletions(-) diff --git a/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh b/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh index 9787a96316..414cf6ef19 100644 --- a/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh +++ b/addons/falkordb/scripts-ut-spec/falkordb_switchover_spec.sh @@ -372,10 +372,16 @@ master_host:redis-master" ORIGINAL_PRIORITIES["redis1"]=100 ORIGINAL_PRIORITIES["redis2"]=100 ORIGINAL_PRIORITIES["redis3"]=0 + redis_replica_announce_address() { + case "$1" in + redis2) printf '%s\t%s\n' "10.0.0.2" "31002" ;; + redis3) printf '%s\t%s\n' "10.0.0.3" "31003" ;; + esac + } sentinel_observed_replica_priority() { - case "$1:$2" in - sentinel1:redis2|sentinel2:redis2) echo "1" ;; - sentinel1:redis3|sentinel2:redis3) echo "0" ;; + case "$1:$2:$3" in + sentinel1:10.0.0.2:31002|sentinel2:10.0.0.2:31002) echo "1" ;; + sentinel1:10.0.0.3:31003|sentinel2:10.0.0.3:31003) echo "0" ;; esac } When call wait_sentinel_sees_priority_bias "redis2" "redis1" @@ -383,7 +389,7 @@ master_host:redis-master" The stdout should include "All Sentinel replica priority caches confirmed" End - It "should parse the target priority from Sentinel replica output" + It "should parse the target priority by its advertised address" export REDIS_CLI_TLS_CMD="" export CUSTOM_SENTINEL_MASTER_NAME="redis" export SENTINEL_SERVICE_PORT="26379" @@ -395,30 +401,21 @@ master_host:redis-master" 3) "slave-priority" 4) "100" 5) "name" -6) "redis2.redis-headless.default.svc.cluster.local:6379" +6) "10.0.0.2:31002" 7) "ip" -8) "redis2.redis-headless.default.svc.cluster.local" -9) "slave-priority" -10) "1" -11) "name" -12) "redis3.redis-headless.default.svc.cluster.local:6379" -13) "slave-priority" -14) "100" +8) "10.0.0.2" +9) "port" +10) "31002" +11) "slave-priority" +12) "1" EOF } - When call sentinel_observed_replica_priority "sentinel1" "redis2.redis-headless.default.svc.cluster.local" + When call sentinel_observed_replica_priority "sentinel1" "10.0.0.2" "31002" The status should be success The output should equal "1" End - It "should not accept a different FQDN with the same pod label" - When call same_fqdn \ - "redis2.other-headless.other.svc.cluster.local" \ - "redis2.redis-headless.default.svc.cluster.local" - The status should be failure - End - - It "should parse an exact target after a same-label collision" + It "should parse an exact target after an address collision" export REDIS_CLI_TLS_CMD="" export CUSTOM_SENTINEL_MASTER_NAME="redis" export SENTINEL_SERVICE_PORT="26379" @@ -426,20 +423,92 @@ EOF redis-cli() { cat <<'EOF' 1) "name" -2) "redis2.other-headless.other.svc.cluster.local:6379" -3) "slave-priority" -4) "100" -5) "name" -6) "redis2.redis-headless.default.svc.cluster.local:6379" +2) "10.0.0.2:31001" +3) "ip" +4) "10.0.0.2" +5) "port" +6) "31001" 7) "slave-priority" -8) "1" +8) "100" +9) "name" +10) "10.0.0.2:31002" +11) "ip" +12) "10.0.0.2" +13) "port" +14) "31002" +15) "slave-priority" +16) "1" EOF } - When call sentinel_observed_replica_priority "sentinel1" "redis2.redis-headless.default.svc.cluster.local" + When call sentinel_observed_replica_priority "sentinel1" "10.0.0.2" "31002" The status should be success The output should equal "1" End + It "should read and resolve an advertised hostname from Redis" + getent() { + printf '%s\n' "10.0.0.2 advertised.example" + } + redis_config_get() { + printf '%s\n' \ + "replica-announce-ip" "advertised.example" \ + "replica-announce-port" "31002" + } + When call redis_replica_announce_address "redis2.redis-headless.default.svc.cluster.local" + The status should be success + The output should equal "$(printf 'advertised.example,10.0.0.2\t31002')" + End + + It "should use the service port for a fixed Pod IP with announce-port zero" + redis_service_port="6379" + redis_config_get() { + printf '%s\n' \ + "replica-announce-ip" "10.0.0.22" \ + "replica-announce-port" "0" + } + When call redis_replica_announce_address "redis2.redis-headless.default.svc.cluster.local" + The status should be success + The output should equal "$(printf '10.0.0.22\t6379')" + End + + It "should reject an exact announce address collision across replicas" + register_exact_collision() { + ANNOUNCE_TUPLE_OWNERS=() + ANNOUNCE_IDENTITY_ERROR="" + register_replica_announce_identity "redis2" "10.0.0.2" "31002" + register_replica_announce_identity "redis3" "10.0.0.2" "31002" + } + When call register_exact_collision + The status should be failure + The stderr should include "Replica announce identity 10.0.0.2:31002 is shared by redis2 and redis3" + End + + It "should reject a resolved address collision across advertised hostnames" + register_alias_collision() { + ANNOUNCE_TUPLE_OWNERS=() + ANNOUNCE_IDENTITY_ERROR="" + register_replica_announce_identity \ + "redis2" "replica-a.example,10.0.0.9" "31002" + register_replica_announce_identity \ + "redis3" "replica-b.example,10.0.0.9" "31002" + } + When call register_alias_collision + The status should be failure + The stderr should include "Replica announce identity 10.0.0.9:31002 is shared by redis2 and redis3" + End + + It "should allow replicas to share an address when their ports differ" + register_distinct_ports() { + ANNOUNCE_TUPLE_OWNERS=() + ANNOUNCE_IDENTITY_ERROR="" + register_replica_announce_identity "redis2" "10.0.0.2" "31002" + register_replica_announce_identity "redis3" "10.0.0.2" "31003" + } + When call register_distinct_ports + The status should be success + The stderr should equal "" + End + It "should reject a requested candidate absent from the exact pod list" export REDIS_POD_FQDN_LIST="redis1.redis-headless.default.svc.cluster.local,redis2.redis-headless.default.svc.cluster.local" export KB_SWITCHOVER_CANDIDATE_FQDN="redis2.other-headless.other.svc.cluster.local" diff --git a/addons/falkordb/scripts/falkordb-switchover.sh b/addons/falkordb/scripts/falkordb-switchover.sh index 778434e2d3..927d0a8343 100644 --- a/addons/falkordb/scripts/falkordb-switchover.sh +++ b/addons/falkordb/scripts/falkordb-switchover.sh @@ -21,6 +21,8 @@ test || __() { } declare -A ORIGINAL_PRIORITIES +declare -A ANNOUNCE_TUPLE_OWNERS +ANNOUNCE_IDENTITY_ERROR="" redis_service_port=${SERVICE_PORT:-6379} readonly redis_cli_timeout_seconds=5 readonly switchover_action_timeout_seconds=420 @@ -266,6 +268,16 @@ redis_config_get() { return 0 } +resolve_host_addresses() { + local host="$1" + command -v getent >/dev/null 2>&1 || return 0 + if [[ "$ut_mode" == "true" ]]; then + getent hosts "$host" + else + timeout -k 2 "$redis_cli_timeout_seconds" getent hosts "$host" + fi +} + execute_sentinel_failover() { local master_name=$1 local success=false @@ -293,12 +305,91 @@ execute_sentinel_failover() { return 0 } +redis_replica_announce_address() { + local replica_fqdn="$1" + local output + local announce_host + local announce_hosts + local announce_port + + if ! output=$(redis_config_get \ + "$replica_fqdn" "$redis_service_port" "$REDIS_DEFAULT_PASSWORD" \ + "CONFIG GET replica-announce-*"); then + echo "Error: Failed to get replica announce address for $replica_fqdn" >&2 + return 1 + fi + + announce_host=$(printf '%s\n' "$output" | awk ' + previous == "replica-announce-ip" { print; exit } + { previous = $0 } + ') + announce_port=$(printf '%s\n' "$output" | awk ' + previous == "replica-announce-port" { print; exit } + { previous = $0 } + ') + + if [[ -z "$announce_host" ]]; then + echo "Error: Empty replica-announce-ip for $replica_fqdn" >&2 + return 1 + fi + if [[ ! "$announce_port" =~ ^[0-9]+$ ]]; then + echo "Error: Invalid replica-announce-port for $replica_fqdn: $announce_port" >&2 + return 1 + fi + if [[ "$announce_port" == "0" ]]; then + announce_port="$redis_service_port" + fi + + announce_hosts=$(normalize_fqdn "$announce_host") + local resolution_output + resolution_output=$(resolve_host_addresses "$announce_host" 2>/dev/null) || true + local resolved_host + while read -r resolved_host _; do + [[ -n "$resolved_host" ]] || continue + if ! fqdn_in_csv "$resolved_host" "$announce_hosts"; then + announce_hosts+=",$(normalize_fqdn "$resolved_host")" + fi + done <<< "$resolution_output" + + printf '%s\t%s\n' "$announce_hosts" "$announce_port" +} + +register_replica_announce_identity() { + local replica_fqdn="$1" + local announce_hosts_csv="$2" + local announce_port="$3" + local -a announce_hosts + local announce_host + local normalized_host + local identity_key + local current_owner + + IFS=',' read -ra announce_hosts <<< "$announce_hosts_csv" + for announce_host in "${announce_hosts[@]}"; do + normalized_host=$(normalize_fqdn "$announce_host") + [[ -n "$normalized_host" ]] || continue + identity_key="${normalized_host}|port=${announce_port}" + current_owner="${ANNOUNCE_TUPLE_OWNERS[$identity_key]:-}" + if [[ -n "$current_owner" ]] && ! same_fqdn "$current_owner" "$replica_fqdn"; then + ANNOUNCE_IDENTITY_ERROR="Replica announce identity ${normalized_host}:${announce_port} is shared by $current_owner and $replica_fqdn" + echo "Error: $ANNOUNCE_IDENTITY_ERROR" >&2 + return 1 + fi + done + + for announce_host in "${announce_hosts[@]}"; do + normalized_host=$(normalize_fqdn "$announce_host") + [[ -n "$normalized_host" ]] || continue + identity_key="${normalized_host}|port=${announce_port}" + ANNOUNCE_TUPLE_OWNERS["$identity_key"]="$replica_fqdn" + done +} + sentinel_observed_replica_priority() { local sentinel_fqdn="$1" - local replica_fqdn="$2" + local replica_announce_hosts="$2" + local replica_announce_port="$3" local master_name="${CUSTOM_SENTINEL_MASTER_NAME:-$REDIS_COMPONENT_NAME}" - local replica_identity - replica_identity=$(normalize_fqdn "$replica_fqdn") local output unset_xtrace_when_ut_mode_false @@ -316,14 +407,32 @@ sentinel_observed_replica_priority() { printf '%s\n' "$output" \ | tr -d '"' \ | sed 's/.*) //' \ - | awk -v candidate="$replica_identity" ' + | awk \ + -v candidate_hosts="$replica_announce_hosts" \ + -v candidate_port="$replica_announce_port" ' + BEGIN { + candidate_count = split(candidate_hosts, host_list, ",") + for (i = 1; i <= candidate_count; i++) { + accepted_hosts[host_list[i]] = 1 + } + } previous == "name" { - identity = tolower($0) - sub(/:[0-9]+$/, "", identity) - sub(/\.$/, "", identity) - in_candidate = (identity == candidate) + replica_host = "" + replica_port = "" + } + previous == "ip" { + replica_host = tolower($0) + sub(/\.$/, "", replica_host) + } + previous == "port" { + replica_port = $0 + } + previous == "slave-priority" { + if (accepted_hosts[replica_host] && replica_port == candidate_port) { + print + exit + } } - in_candidate && previous == "slave-priority" { print; exit } { previous = $0 } ' } @@ -337,9 +446,31 @@ wait_sentinel_sees_priority_bias() { while [[ $SECONDS -lt $deadline ]]; do IFS=',' read -ra sentinel_pod_fqdn_list <<< "${SENTINEL_POD_FQDN_LIST}" IFS=',' read -ra redis_pod_fqdn_list <<< "${REDIS_POD_FQDN_LIST}" + local -A announce_hosts=() + local -A announce_ports=() + local announce_identity_conflict=false + ANNOUNCE_TUPLE_OWNERS=() + ANNOUNCE_IDENTITY_ERROR="" + local redis_pod_fqdn + for redis_pod_fqdn in "${redis_pod_fqdn_list[@]}"; do + same_fqdn "$redis_pod_fqdn" "$current_master" && continue + local announce_address + if announce_address=$(redis_replica_announce_address "$redis_pod_fqdn"); then + IFS=$'\t' read -r \ + announce_hosts["$redis_pod_fqdn"] \ + announce_ports["$redis_pod_fqdn"] <<< "$announce_address" + if ! register_replica_announce_identity \ + "$redis_pod_fqdn" \ + "${announce_hosts[$redis_pod_fqdn]}" \ + "${announce_ports[$redis_pod_fqdn]}" 2>/dev/null; then + announce_identity_conflict=true + fi + fi + done + local total=0 local confirmed=0 - local sentinel_pod_fqdn redis_pod_fqdn + local sentinel_pod_fqdn for sentinel_pod_fqdn in "${sentinel_pod_fqdn_list[@]}"; do for redis_pod_fqdn in "${redis_pod_fqdn_list[@]}"; do @@ -354,19 +485,26 @@ wait_sentinel_sees_priority_bias() { && expected_priority=0 total=$((total + 1)) + [[ -n "${announce_hosts[$redis_pod_fqdn]:-}" ]] || continue observed_priority=$(sentinel_observed_replica_priority \ - "$sentinel_pod_fqdn" "$redis_pod_fqdn") || true + "$sentinel_pod_fqdn" \ + "${announce_hosts[$redis_pod_fqdn]}" \ + "${announce_ports[$redis_pod_fqdn]}") || true [[ "$observed_priority" == "$expected_priority" ]] && confirmed=$((confirmed + 1)) done done - if [[ $total -gt 0 && $confirmed -eq $total ]]; then + if [[ "$announce_identity_conflict" == "false" \ + && $total -gt 0 \ + && $confirmed -eq $total ]]; then echo "All Sentinel replica priority caches confirmed targeted bias for $candidate_fqdn." return 0 fi sleep_when_ut_mode_false 1 done + [[ -z "$ANNOUNCE_IDENTITY_ERROR" ]] \ + || echo "Error: $ANNOUNCE_IDENTITY_ERROR" >&2 echo "Error: Sentinel did not confirm targeted priority bias for $candidate_fqdn within ${sentinel_priority_wait_seconds}s" >&2 return 1 } From e1c623d855b358e7d7cc5a1cb36a2ebc4e22e817 Mon Sep 17 00:00:00 2001 From: wei Date: Fri, 31 Jul 2026 13:19:41 +0800 Subject: [PATCH 3/4] fix(falkordb): version switchover timeout upgrade --- addons/falkordb/Chart.yaml | 2 +- addons/falkordb/README.md | 27 ++++ .../examples/upgrade-switchover-timeout.sh | 145 ++++++++++++++++++ .../examples/upgrade-switchover-timeout.yaml | 12 ++ addons/falkordb/releases_notes.yaml | 9 ++ .../switchover_timeout_contract_spec.sh | 53 ++++++- .../upgrade_switchover_timeout_spec.sh | 132 ++++++++++++++++ 7 files changed, 373 insertions(+), 7 deletions(-) create mode 100755 addons/falkordb/examples/upgrade-switchover-timeout.sh create mode 100644 addons/falkordb/examples/upgrade-switchover-timeout.yaml create mode 100644 addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh diff --git a/addons/falkordb/Chart.yaml b/addons/falkordb/Chart.yaml index c71c5b3277..e533c9fc71 100644 --- a/addons/falkordb/Chart.yaml +++ b/addons/falkordb/Chart.yaml @@ -4,7 +4,7 @@ description: "FalkorDB is an in-memory graph database based on Redis." type: application -version: 1.2.0-alpha.0 +version: 1.2.0-alpha.1 appVersion: "4.12.5" diff --git a/addons/falkordb/README.md b/addons/falkordb/README.md index ef512cb000..fcfec88946 100644 --- a/addons/falkordb/README.md +++ b/addons/falkordb/README.md @@ -40,6 +40,33 @@ FalkorDB is an open source (SSPL licensed) in-memory graph database based on Red ## Examples +### Upgrade an existing Cluster to the bounded switchover definition + +FalkorDB addon `1.2.0-alpha.1` publishes the standalone and replication +FalkorDB component as `falkordb-4-1.2.0-alpha.1`. This new definition disables +kbagent's 30-second default action timeout while the switchover script enforces +its own bounded deadline. + +Upgrading the addon definitions does not change an existing Cluster's resolved +ComponentDefinition or restart its Pods. After upgrading the addon, edit the +namespace, Cluster name, and OpsRequest name in +`examples/upgrade-switchover-timeout.yaml`, then run the bounded migration +verifier: + +```bash +./examples/upgrade-switchover-timeout.sh +``` + +The OpsRequest changes only the `falkordb` component to +`falkordb-4-1.2.0-alpha.1`; KubeBlocks then recreates its Pods from the new +definition. Do not patch the resolved Component directly or add `force: true`. +The verifier does not trust the OpsRequest phase by itself. It returns success +only after the OpsRequest is `Succeed`, the Component references the new +definition, every original Pod UID is gone, every replacement Pod is `Ready`, +and every replacement kbagent has the serialized switchover timeout `-1`. +It fails after 10 minutes unless `UPGRADE_VERIFY_TIMEOUT_SECONDS` overrides the +bounded deadline. Request a switchover only after this verification succeeds. + ### Create Create a FalkorDB replication cluster with two components, one for FalkorDB, and one for Sentinel[^1]. diff --git a/addons/falkordb/examples/upgrade-switchover-timeout.sh b/addons/falkordb/examples/upgrade-switchover-timeout.sh new file mode 100755 index 0000000000..f1997b8f73 --- /dev/null +++ b/addons/falkordb/examples/upgrade-switchover-timeout.sh @@ -0,0 +1,145 @@ +#!/usr/bin/env bash + +set -euo pipefail + +manifest="${1:-$(dirname "$0")/upgrade-switchover-timeout.yaml}" +verify_timeout="${UPGRADE_VERIFY_TIMEOUT_SECONDS:-600}" +poll_interval="${UPGRADE_VERIFY_POLL_SECONDS:-5}" +api_timeout="${UPGRADE_VERIFY_API_TIMEOUT_SECONDS:-10}" + +for command in kubectl jq; do + if ! command -v "$command" >/dev/null 2>&1; then + printf 'required command not found: %s\n' "$command" >&2 + exit 1 + fi +done + +if [[ ! "$verify_timeout" =~ ^[1-9][0-9]*$ ]] || + [[ ! "$poll_interval" =~ ^[1-9][0-9]*$ ]] || + [[ ! "$api_timeout" =~ ^[1-9][0-9]*$ ]]; then + printf 'verification, polling, and API timeouts must be positive integers\n' >&2 + exit 1 +fi + +if [[ "${FALKORDB_UPGRADE_VERIFY_SUPERVISED:-0}" != "1" ]]; then + timeout_command="$(command -v timeout || command -v gtimeout || true)" + if [[ -z "$timeout_command" ]]; then + printf 'required command not found: timeout or gtimeout\n' >&2 + exit 1 + fi + export FALKORDB_UPGRADE_VERIFY_SUPERVISED=1 + set +e + "$timeout_command" --signal=TERM --kill-after=10s "${verify_timeout}s" \ + "$BASH" "$0" "$@" + status=$? + set -e + if ((status == 124 || status == 137)); then + printf 'timed out verifying ComponentDefinition and recreated kbagent Pods\n' >&2 + fi + exit "$status" +fi + +kubectl_timeout="--request-timeout=${api_timeout}s" +ops_json="$(kubectl "$kubectl_timeout" create --dry-run=client -o json -f "$manifest")" +namespace="$(jq -er '.metadata.namespace // "default"' <<<"$ops_json")" +ops_name="$(jq -er '.metadata.name' <<<"$ops_json")" +cluster_name="$(jq -er '.spec.clusterName' <<<"$ops_json")" +component_name="$(jq -er '.spec.upgrade.components[0].componentName' <<<"$ops_json")" +target_comp_def="$(jq -er '.spec.upgrade.components[0].componentDefinitionName' <<<"$ops_json")" + +jq -e ' + .kind == "OpsRequest" and + .spec.type == "Upgrade" and + (.spec.force // false) == false and + (.spec.upgrade.components | length) == 1 and + (.spec.upgrade.components[0] | has("serviceVersion") | not) +' <<<"$ops_json" >/dev/null + +selector="app.kubernetes.io/instance=${cluster_name},apps.kubeblocks.io/component-name=${component_name}" +old_pods_json="$(kubectl "$kubectl_timeout" get pods -n "$namespace" -l "$selector" -o json)" +old_pod_count="$(jq -er '.items | length' <<<"$old_pods_json")" +if ((old_pod_count == 0)); then + printf 'no existing Pods match %s\n' "$selector" >&2 + exit 1 +fi +old_pod_uids="$(jq -c '[.items[].metadata.uid]' <<<"$old_pods_json")" + +kubectl "$kubectl_timeout" create -f "$manifest" + +deadline=$((SECONDS + verify_timeout)) +while ((SECONDS < deadline)); do + ops_phase="$(kubectl "$kubectl_timeout" get opsrequest "$ops_name" -n "$namespace" -o jsonpath='{.status.phase}' 2>/dev/null || true)" + if [[ "$ops_phase" == "Failed" || "$ops_phase" == "Aborted" || "$ops_phase" == "Cancelled" ]]; then + printf 'Upgrade OpsRequest entered terminal phase %s\n' "$ops_phase" >&2 + exit 1 + fi + + components_json="$(kubectl "$kubectl_timeout" get components.apps.kubeblocks.io -n "$namespace" -l "$selector" -o json 2>/dev/null || true)" + pods_json="$(kubectl "$kubectl_timeout" get pods -n "$namespace" -l "$selector" -o json 2>/dev/null || true)" + if [[ -z "$components_json" || -z "$pods_json" ]]; then + sleep "$poll_interval" + continue + fi + + if [[ "$ops_phase" == "Succeed" ]] && + jq -e --arg target "$target_comp_def" ' + (.items | length) == 1 and + .items[0].spec.compDef == $target + ' <<<"$components_json" >/dev/null && + jq -e --argjson old_uids "$old_pod_uids" --argjson expected_count "$old_pod_count" ' + (.items | length) == $expected_count and + all(.items[]; + .metadata.deletionTimestamp == null and + (.metadata.uid as $uid | ($old_uids | index($uid)) == null) and + any(.status.conditions[]?; .type == "Ready" and .status == "True") and + ([ + .spec.containers[] + | select(.name == "kbagent") + | .env[]? + | select(.name == "KB_AGENT_ACTION") + | .value + ] | length) == 1 and + ([ + .spec.containers[] + | select(.name == "kbagent") + | .env[]? + | select(.name == "KB_AGENT_ACTION") + | .value + | fromjson + | .[] + | select(.name == "switchover") + ] | length) == 1 and + all( + .spec.containers[] + | select(.name == "kbagent") + | .env[]? + | select(.name == "KB_AGENT_ACTION") + | .value + | fromjson + | .[] + | select(.name == "switchover"); + .timeoutSeconds == -1 + ) + ) + ' <<<"$pods_json" >/dev/null; then + printf 'Upgrade verified: component=%s compDef=%s recreatedPods=%s\n' \ + "$component_name" "$target_comp_def" "$old_pod_count" + exit 0 + fi + + remaining=$((deadline - SECONDS)) + if ((remaining <= 0)); then + break + fi + sleep_interval="$poll_interval" + if ((sleep_interval > remaining)); then + sleep_interval="$remaining" + fi + sleep "$sleep_interval" +done + +printf 'timed out verifying ComponentDefinition and recreated kbagent Pods\n' >&2 +kubectl "$kubectl_timeout" get opsrequest "$ops_name" -n "$namespace" -o wide >&2 || true +kubectl "$kubectl_timeout" get components.apps.kubeblocks.io -n "$namespace" -l "$selector" -o wide >&2 || true +kubectl "$kubectl_timeout" get pods -n "$namespace" -l "$selector" -o wide >&2 || true +exit 1 diff --git a/addons/falkordb/examples/upgrade-switchover-timeout.yaml b/addons/falkordb/examples/upgrade-switchover-timeout.yaml new file mode 100644 index 0000000000..bb3e9b9154 --- /dev/null +++ b/addons/falkordb/examples/upgrade-switchover-timeout.yaml @@ -0,0 +1,12 @@ +apiVersion: operations.kubeblocks.io/v1alpha1 +kind: OpsRequest +metadata: + name: falkordb-switchover-timeout-upgrade + namespace: demo +spec: + clusterName: falkordb-replication + type: Upgrade + upgrade: + components: + - componentName: falkordb + componentDefinitionName: falkordb-4-1.2.0-alpha.1 diff --git a/addons/falkordb/releases_notes.yaml b/addons/falkordb/releases_notes.yaml index e69de29bb2..0c78c43682 100644 --- a/addons/falkordb/releases_notes.yaml +++ b/addons/falkordb/releases_notes.yaml @@ -0,0 +1,9 @@ +releases: + - version: "1.2.0-alpha.1" + released_at: "" + status: "alpha" + notes: "Bound standalone switchover execution and publish it under a new ComponentDefinition identity. Existing standalone and replication Clusters must use the shipped Upgrade OpsRequest and bounded readback verifier to recreate the FalkorDB Pods with the new kbagent action timeout." + git_branch: "main" + git_tag: "" + commit_sha: "" + breaking_changes: "The FalkorDB ComponentDefinition name changes from falkordb-4-1.2.0-alpha.0 to falkordb-4-1.2.0-alpha.1. Addon upgrade alone does not migrate existing Clusters." diff --git a/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh b/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh index 9e74b5f587..51d1ed5f5b 100644 --- a/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh +++ b/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh @@ -5,27 +5,68 @@ Describe "FalkorDB switchover timeout contract" printf '%s/addons/falkordb\n' "$(cd "${SHELLSPEC_PROJECT_ROOT:-.}" && pwd)" } - rendered_switchover_timeout() { + rendered_upgrade_contract() { helm template test "$(chart_path)" \ --show-only templates/cmpd-falkordb.yaml \ | ruby -ryaml -e ' component = YAML.load_stream($stdin.read).compact.find do |doc| doc["kind"] == "ComponentDefinition" && - doc.dig("metadata", "name") == "falkordb-4-1.2.0-alpha.0" + doc.dig("metadata", "name") == "falkordb-4-1.2.0-alpha.1" end abort "rendered ComponentDefinition not found" unless component timeout = component.dig( "spec", "lifecycleActions", "switchover", "timeoutSeconds" ) abort "rendered switchover timeout is #{timeout.inspect}" unless timeout == -1 - puts timeout + puts "definition=#{component.dig("metadata", "name")},timeout=#{timeout}" ' + + ruby -ryaml -e ' + migration = YAML.load_file( + File.join(ARGV.fetch(0), "examples", "upgrade-switchover-timeout.yaml") + ) + components = migration.dig("spec", "upgrade", "components") + abort "expected exactly one upgrade component" unless components&.length == 1 + component = components.first + force = migration.dig("spec", "force") + abort "migration must not force the upgrade" unless force.nil? || force == false + abort "migration must preserve serviceVersion" if component.key?("serviceVersion") + puts [ + "migration=#{migration.dig("spec", "type")}", + "component=#{component["componentName"]}", + "target=#{component["componentDefinitionName"]}", + "serviceVersion=preserved", + "force=false" + ].join(",") + ' "$(chart_path)" + + ruby -e ' + verifier = File.read( + File.join(ARGV.fetch(0), "examples", "upgrade-switchover-timeout.sh") + ) + required = [ + "old_pod_uids", + ".items[0].spec.compDef == $target", + "($old_uids | index($uid)) == null", + ".metadata.deletionTimestamp == null", + ".name == \"KB_AGENT_ACTION\"", + ".timeoutSeconds == -1", + "--request-timeout=", + "--kill-after=10s", + "SECONDS < deadline" + ] + missing = required.reject { |contract| verifier.include?(contract) } + abort "migration verifier missing: #{missing.join(", ")}" unless missing.empty? + puts "readback=component+pod-uid+ready+kbagent,bounded=true" + ' "$(chart_path)" } - It "renders the negative kbagent timeout on the intended lifecycle action" - When call rendered_switchover_timeout + It "publishes a new definition and an explicit existing-cluster upgrade" + When call rendered_upgrade_contract The status should be success - The output should equal "-1" + The output should equal "definition=falkordb-4-1.2.0-alpha.1,timeout=-1 +migration=Upgrade,component=falkordb,target=falkordb-4-1.2.0-alpha.1,serviceVersion=preserved,force=false +readback=component+pod-uid+ready+kbagent,bounded=true" End End diff --git a/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh b/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh new file mode 100644 index 0000000000..90d049b8dd --- /dev/null +++ b/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh @@ -0,0 +1,132 @@ +# shellcheck shell=bash + +Describe "FalkorDB switchover timeout upgrade verifier" + setup_mock_kubectl() { + chart_dir="$(cd "${SHELLSPEC_PROJECT_ROOT:-.}" && pwd)/addons/falkordb" + mock_bin_dir="$(mktemp -d)" + pod_read_count_file="$mock_bin_dir/pod-read-count" + original_path="$PATH" + + cat >"$mock_bin_dir/kubectl" <<'MOCK' +#!/usr/bin/env bash +set -euo pipefail + +if [[ "$*" == *"create --dry-run=client -o json"* ]]; then + if [[ "${STALL_PREFLIGHT:-0}" == "1" ]]; then + sleep 5 + fi + cat <<'JSON' +{"kind":"OpsRequest","metadata":{"name":"upgrade","namespace":"demo"},"spec":{"clusterName":"cluster","type":"Upgrade","upgrade":{"components":[{"componentName":"falkordb","componentDefinitionName":"falkordb-4-1.2.0-alpha.1"}]}}} +JSON +elif [[ "$*" == *" create -f "* ]]; then + printf 'opsrequest.operations.kubeblocks.io/upgrade created\n' +elif [[ "$*" == *"get opsrequest"* ]]; then + printf 'Succeed' +elif [[ "$*" == *"get components.apps.kubeblocks.io"* ]]; then + cat <<'JSON' +{"items":[{"spec":{"compDef":"falkordb-4-1.2.0-alpha.1"}}]} +JSON +elif [[ "$*" == *"get pods"* ]]; then + count=0 + if [[ -f "$POD_READ_COUNT_FILE" ]]; then + count="$(cat "$POD_READ_COUNT_FILE")" + fi + count=$((count + 1)) + printf '%s\n' "$count" >"$POD_READ_COUNT_FILE" + if ((count == 1 || ${KEEP_OLD_POD:-0} == 1)); then + cat <<'JSON' +{"items":[{"metadata":{"uid":"old-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":0}]"}]}]}}]} +JSON + elif [[ "${CONFLICTING_ACTION:-0}" == "1" ]]; then + cat <<'JSON' +{"items":[{"metadata":{"uid":"new-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":-1},{\"name\":\"switchover\",\"timeoutSeconds\":0}]"}]}]}}]} +JSON + elif [[ "${TERMINATING_POD:-0}" == "1" ]]; then + cat <<'JSON' +{"items":[{"metadata":{"uid":"new-uid","deletionTimestamp":"2026-07-31T00:00:00Z"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":-1}]"}]}]}}]} +JSON + else + cat <<'JSON' +{"items":[{"metadata":{"uid":"new-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":-1}]"}]}]}}]} +JSON + fi +else + printf 'unexpected kubectl call: %s\n' "$*" >&2 + exit 1 +fi +MOCK + chmod +x "$mock_bin_dir/kubectl" + export PATH="$mock_bin_dir:$PATH" + export POD_READ_COUNT_FILE="$pod_read_count_file" + export UPGRADE_VERIFY_TIMEOUT_SECONDS=2 + export UPGRADE_VERIFY_POLL_SECONDS=1 + export UPGRADE_VERIFY_API_TIMEOUT_SECONDS=1 + } + + cleanup_mock_kubectl() { + PATH="$original_path" + rm -rf "$mock_bin_dir" + unset chart_dir mock_bin_dir pod_read_count_file original_path + unset POD_READ_COUNT_FILE KEEP_OLD_POD CONFLICTING_ACTION TERMINATING_POD STALL_PREFLIGHT + unset UPGRADE_VERIFY_TIMEOUT_SECONDS UPGRADE_VERIFY_POLL_SECONDS + unset UPGRADE_VERIFY_API_TIMEOUT_SECONDS FALKORDB_UPGRADE_VERIFY_SUPERVISED + } + + BeforeEach "setup_mock_kubectl" + AfterEach "cleanup_mock_kubectl" + + It "accepts only a new ready Pod with the target kbagent timeout" + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be success + The stdout should include "Upgrade verified" + The stdout should include "recreatedPods=1" + End + + It "fails closed when the old Pod UID remains" + export KEEP_OLD_POD=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "fails closed when switchover actions conflict" + export CONFLICTING_ACTION=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "fails closed when the replacement Pod is terminating" + export TERMINATING_POD=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "caps an oversized poll interval with the wall-clock deadline" + export KEEP_OLD_POD=1 + export UPGRADE_VERIFY_TIMEOUT_SECONDS=1 + export UPGRADE_VERIFY_POLL_SECONDS=99 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "bounds a stalled preflight API call with the wall-clock deadline" + export STALL_PREFLIGHT=1 + export UPGRADE_VERIFY_TIMEOUT_SECONDS=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End +End From ebfde18310304ffd1c6044a5c7e373604abc5684 Mon Sep 17 00:00:00 2001 From: wei Date: Fri, 31 Jul 2026 14:20:39 +0800 Subject: [PATCH 4/4] fix(falkordb): preserve upgrade dependencies --- addons/falkordb/README.md | 31 +- .../examples/upgrade-switchover-timeout.sh | 232 +++++++++++++- .../examples/upgrade-switchover-timeout.yaml | 1 + addons/falkordb/releases_notes.yaml | 2 +- .../switchover_timeout_contract_spec.sh | 34 ++- .../upgrade_switchover_timeout_spec.sh | 289 ++++++++++++++++-- addons/falkordb/templates/cmpd-falkordb.yaml | 12 +- 7 files changed, 542 insertions(+), 59 deletions(-) diff --git a/addons/falkordb/README.md b/addons/falkordb/README.md index fcfec88946..e702d19d4f 100644 --- a/addons/falkordb/README.md +++ b/addons/falkordb/README.md @@ -49,22 +49,33 @@ its own bounded deadline. Upgrading the addon definitions does not change an existing Cluster's resolved ComponentDefinition or restart its Pods. After upgrading the addon, edit the -namespace, Cluster name, and OpsRequest name in +namespace, Cluster name, OpsRequest name, and `serviceVersion` in `examples/upgrade-switchover-timeout.yaml`, then run the bounded migration -verifier: +verifier. The example pins `4.12.5`; the value must exactly equal the live +FalkorDB Component's current `spec.serviceVersion`. ```bash ./examples/upgrade-switchover-timeout.sh ``` -The OpsRequest changes only the `falkordb` component to -`falkordb-4-1.2.0-alpha.1`; KubeBlocks then recreates its Pods from the new -definition. Do not patch the resolved Component directly or add `force: true`. -The verifier does not trust the OpsRequest phase by itself. It returns success -only after the OpsRequest is `Succeed`, the Component references the new -definition, every original Pod UID is gone, every replacement Pod is `Ready`, -and every replacement kbagent has the serialized switchover timeout `-1`. -It fails after 10 minutes unless `UPGRADE_VERIFY_TIMEOUT_SECONDS` overrides the +The OpsRequest changes only the `falkordb` component definition. The new main +definition uses the stable `falkordb-sent-4` prefix for all Sentinel references, +so an existing replication Cluster keeps its authoritative Sentinel sibling +without recreating it. Do not patch the resolved Component directly or add +`force: true`. + +The verifier does not trust the OpsRequest phase by itself. Before creation it +rejects a manifest whose explicit `serviceVersion` differs from the live +Component. It returns success only after the OpsRequest is `Succeed`, the +Component references the new definition and the same service version, every +original Pod UID is gone, every replacement Pod is `Ready`, Pod image names and +image IDs are unchanged, and every replacement kbagent has the serialized +switchover timeout `-1`. For replication, the old main Pods must agree on all +six Sentinel variables, exactly one non-terminating Sentinel sibling must match +that contract, the same sibling name and UID must remain after the upgrade, and +every replacement main Pod must preserve the six variables exactly. Standalone +Pods must have neither that environment contract nor a Sentinel sibling. It +fails after 10 minutes unless `UPGRADE_VERIFY_TIMEOUT_SECONDS` overrides the bounded deadline. Request a switchover only after this verification succeeds. ### Create diff --git a/addons/falkordb/examples/upgrade-switchover-timeout.sh b/addons/falkordb/examples/upgrade-switchover-timeout.sh index f1997b8f73..5232732fa3 100755 --- a/addons/falkordb/examples/upgrade-switchover-timeout.sh +++ b/addons/falkordb/examples/upgrade-switchover-timeout.sh @@ -47,15 +47,42 @@ cluster_name="$(jq -er '.spec.clusterName' <<<"$ops_json")" component_name="$(jq -er '.spec.upgrade.components[0].componentName' <<<"$ops_json")" target_comp_def="$(jq -er '.spec.upgrade.components[0].componentDefinitionName' <<<"$ops_json")" -jq -e ' +if ! jq -e ' .kind == "OpsRequest" and .spec.type == "Upgrade" and (.spec.force // false) == false and (.spec.upgrade.components | length) == 1 and - (.spec.upgrade.components[0] | has("serviceVersion") | not) -' <<<"$ops_json" >/dev/null + (.spec.upgrade.components[0].serviceVersion | type == "string" and length > 0) +' <<<"$ops_json" >/dev/null; then + printf 'Upgrade manifest must contain one non-forced component with an explicit serviceVersion\n' >&2 + exit 1 +fi +target_service_version="$(jq -er '.spec.upgrade.components[0].serviceVersion' <<<"$ops_json")" selector="app.kubernetes.io/instance=${cluster_name},apps.kubeblocks.io/component-name=${component_name}" +cluster_selector="app.kubernetes.io/instance=${cluster_name}" +sentinel_env_names='[ + "SENTINEL_COMPONENT_NAME", + "SENTINEL_USER", + "SENTINEL_PASSWORD", + "SENTINEL_POD_NAME_LIST", + "SENTINEL_POD_FQDN_LIST", + "SENTINEL_SERVICE_PORT" +]' +current_components_json="$(kubectl "$kubectl_timeout" get components.apps.kubeblocks.io -n "$namespace" -l "$selector" -o json)" +current_service_version="$(jq -er ' + if (.items | length) == 1 then + .items[0].spec.serviceVersion + else + error("expected exactly one live Component") + end +' <<<"$current_components_json")" +if [[ "$current_service_version" != "$target_service_version" ]]; then + printf 'manifest serviceVersion %s does not match live Component serviceVersion %s\n' \ + "$target_service_version" "$current_service_version" >&2 + exit 1 +fi + old_pods_json="$(kubectl "$kubectl_timeout" get pods -n "$namespace" -l "$selector" -o json)" old_pod_count="$(jq -er '.items | length' <<<"$old_pods_json")" if ((old_pod_count == 0)); then @@ -64,6 +91,148 @@ if ((old_pod_count == 0)); then fi old_pod_uids="$(jq -c '[.items[].metadata.uid]' <<<"$old_pods_json")" +pod_image_contract() { + jq -c ' + [ + .items[] + | { + name: .metadata.name, + spec: ( + ( + [.spec.initContainers[]? | {kind: "init", name, image}] + + [.spec.containers[]? | {kind: "container", name, image}] + ) | sort_by(.kind, .name) + ), + status: ( + ( + [.status.initContainerStatuses[]? | {kind: "init", name, image, imageID}] + + [.status.containerStatuses[]? | {kind: "container", name, image, imageID}] + ) | sort_by(.kind, .name) + ) + } + ] | sort_by(.name) + ' +} + +if ! jq -e ' + all(.items[]; + .metadata.deletionTimestamp == null and + any(.status.conditions[]?; .type == "Ready" and .status == "True") and + ([.spec.containers[]? | select(.name == "falkordb")] | length) == 1 and + ( + ( + [.spec.initContainers[]? | ["init", .name]] + + [.spec.containers[]? | ["container", .name]] + ) | sort + ) == + ( + ( + [.status.initContainerStatuses[]? | ["init", .name]] + + [.status.containerStatuses[]? | ["container", .name]] + ) | sort + ) and + all( + (.status.initContainerStatuses[]?, .status.containerStatuses[]?); + (.image // "") != "" and (.imageID // "") != "" + ) + ) +' <<<"$old_pods_json" >/dev/null; then + printf 'existing Pods must be non-terminating, Ready, and expose resolved image IDs\n' >&2 + exit 1 +fi +old_image_contract="$(pod_image_contract <<<"$old_pods_json")" + +if ! jq -e --argjson names "$sentinel_env_names" ' + def sentinel_env_contract($pod): + [ + $pod.spec.containers[] + | select(.name == "falkordb") + | .env[]? + | select(.name as $name | ($names | index($name)) != null) + | { + name, + value: (.value // null), + valueFrom: (.valueFrom // null) + } + ] | sort_by(.name); + + [.items[] | sentinel_env_contract(.)] as $contracts | + ($contracts | length) > 0 and + all($contracts[]; + length == 0 or + ( + length == ($names | length) and + ([.[].name] | unique | length) == ($names | length) and + all(.[]; + ((.value // "") != "") or + (.valueFrom != null) + ) + ) + ) and + all($contracts[]; . == $contracts[0]) +' <<<"$old_pods_json" >/dev/null; then + printf 'existing Pods have incomplete, duplicate, empty, or inconsistent Sentinel environment contracts\n' >&2 + exit 1 +fi +old_sentinel_env_contract="$(jq -c --argjson names "$sentinel_env_names" ' + [ + .items[0].spec.containers[] + | select(.name == "falkordb") + | .env[]? + | select(.name as $name | ($names | index($name)) != null) + | { + name, + value: (.value // null), + valueFrom: (.valueFrom // null) + } + ] | sort_by(.name) +' <<<"$old_pods_json")" + +cluster_components_json="$(kubectl "$kubectl_timeout" get components.apps.kubeblocks.io -n "$namespace" -l "$cluster_selector" -o json)" +sentinel_components="$(jq -c ' + [ + .items[] + | select( + (.metadata.labels["apps.kubeblocks.io/component-name"] // "") == "falkordb-sent" or + (.spec.compDef // "" | startswith("falkordb-sent-4")) + ) + | { + name: .metadata.name, + uid: .metadata.uid, + terminating: (.metadata.deletionTimestamp != null) + } + ] | sort_by(.name) +' <<<"$cluster_components_json")" +if ! jq -e ' + length <= 1 and + all(.[]; + .terminating == false and + (.name | type == "string" and length > 0) and + (.uid | type == "string" and length > 0) + ) +' <<<"$sentinel_components" >/dev/null; then + printf 'expected at most one non-terminating FalkorDB Sentinel Component\n' >&2 + exit 1 +fi + +old_requires_sentinel="$(jq -r 'length > 0' <<<"$old_sentinel_env_contract")" +sentinel_component_count="$(jq -r 'length' <<<"$sentinel_components")" +if [[ "$old_requires_sentinel" == "true" && "$sentinel_component_count" != "1" ]]; then + printf 'existing Pods require Sentinel but exactly one live Sentinel Component was not found\n' >&2 + exit 1 +fi +if [[ "$old_requires_sentinel" == "false" && "$sentinel_component_count" != "0" ]]; then + printf 'existing Pods are standalone but a Sentinel Component was found\n' >&2 + exit 1 +fi +expected_sentinel_identity="$(jq -c ' + if length == 1 then + .[0] | {name, uid} + else + null + end +' <<<"$sentinel_components")" + kubectl "$kubectl_timeout" create -f "$manifest" deadline=$((SECONDS + verify_timeout)) @@ -75,23 +244,65 @@ while ((SECONDS < deadline)); do fi components_json="$(kubectl "$kubectl_timeout" get components.apps.kubeblocks.io -n "$namespace" -l "$selector" -o json 2>/dev/null || true)" + cluster_components_json="$(kubectl "$kubectl_timeout" get components.apps.kubeblocks.io -n "$namespace" -l "$cluster_selector" -o json 2>/dev/null || true)" pods_json="$(kubectl "$kubectl_timeout" get pods -n "$namespace" -l "$selector" -o json 2>/dev/null || true)" - if [[ -z "$components_json" || -z "$pods_json" ]]; then + if [[ -z "$components_json" || -z "$cluster_components_json" || -z "$pods_json" ]]; then sleep "$poll_interval" continue fi if [[ "$ops_phase" == "Succeed" ]] && - jq -e --arg target "$target_comp_def" ' + jq -e --arg target "$target_comp_def" --arg service_version "$target_service_version" ' (.items | length) == 1 and - .items[0].spec.compDef == $target + .items[0].spec.compDef == $target and + .items[0].spec.serviceVersion == $service_version ' <<<"$components_json" >/dev/null && - jq -e --argjson old_uids "$old_pod_uids" --argjson expected_count "$old_pod_count" ' + jq -e --argjson expected "$expected_sentinel_identity" ' + [ + .items[] + | select( + (.metadata.labels["apps.kubeblocks.io/component-name"] // "") == "falkordb-sent" or + (.spec.compDef // "" | startswith("falkordb-sent-4")) + ) + | { + name: .metadata.name, + uid: .metadata.uid, + terminating: (.metadata.deletionTimestamp != null) + } + ] as $sentinels | + if $expected == null then + ($sentinels | length) == 0 + else + ($sentinels | length) == 1 and + $sentinels[0].terminating == false and + ($sentinels[0] | {name, uid}) == $expected + end + ' <<<"$cluster_components_json" >/dev/null && + jq -e \ + --argjson old_uids "$old_pod_uids" \ + --argjson expected_count "$old_pod_count" \ + --argjson sentinel_names "$sentinel_env_names" \ + --argjson expected_sentinel_env "$old_sentinel_env_contract" ' + def sentinel_env_contract($pod): + [ + $pod.spec.containers[] + | select(.name == "falkordb") + | .env[]? + | select(.name as $name | ($sentinel_names | index($name)) != null) + | { + name, + value: (.value // null), + valueFrom: (.valueFrom // null) + } + ] | sort_by(.name); + (.items | length) == $expected_count and all(.items[]; + . as $pod | .metadata.deletionTimestamp == null and (.metadata.uid as $uid | ($old_uids | index($uid)) == null) and any(.status.conditions[]?; .type == "Ready" and .status == "True") and + sentinel_env_contract($pod) == $expected_sentinel_env and ([ .spec.containers[] | select(.name == "kbagent") @@ -121,9 +332,10 @@ while ((SECONDS < deadline)); do .timeoutSeconds == -1 ) ) - ' <<<"$pods_json" >/dev/null; then - printf 'Upgrade verified: component=%s compDef=%s recreatedPods=%s\n' \ - "$component_name" "$target_comp_def" "$old_pod_count" + ' <<<"$pods_json" >/dev/null && + [[ "$(pod_image_contract <<<"$pods_json")" == "$old_image_contract" ]]; then + printf 'Upgrade verified: component=%s compDef=%s serviceVersion=%s recreatedPods=%s images=unchanged\n' \ + "$component_name" "$target_comp_def" "$target_service_version" "$old_pod_count" exit 0 fi diff --git a/addons/falkordb/examples/upgrade-switchover-timeout.yaml b/addons/falkordb/examples/upgrade-switchover-timeout.yaml index bb3e9b9154..24bd5aa367 100644 --- a/addons/falkordb/examples/upgrade-switchover-timeout.yaml +++ b/addons/falkordb/examples/upgrade-switchover-timeout.yaml @@ -10,3 +10,4 @@ spec: components: - componentName: falkordb componentDefinitionName: falkordb-4-1.2.0-alpha.1 + serviceVersion: "4.12.5" diff --git a/addons/falkordb/releases_notes.yaml b/addons/falkordb/releases_notes.yaml index 0c78c43682..244dc56d61 100644 --- a/addons/falkordb/releases_notes.yaml +++ b/addons/falkordb/releases_notes.yaml @@ -2,7 +2,7 @@ releases: - version: "1.2.0-alpha.1" released_at: "" status: "alpha" - notes: "Bound standalone switchover execution and publish it under a new ComponentDefinition identity. Existing standalone and replication Clusters must use the shipped Upgrade OpsRequest and bounded readback verifier to recreate the FalkorDB Pods with the new kbagent action timeout." + notes: "Bound standalone switchover execution and publish it under a new ComponentDefinition identity. Existing standalone and replication Clusters must use the shipped version-pinned Upgrade OpsRequest and bounded readback verifier to recreate the FalkorDB Pods with the new kbagent action timeout. The new main definition resolves any same-major Sentinel sibling through the stable falkordb-sent-4 prefix." git_branch: "main" git_tag: "" commit_sha: "" diff --git a/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh b/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh index 51d1ed5f5b..73459ea0f0 100644 --- a/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh +++ b/addons/falkordb/scripts-ut-spec/switchover_timeout_contract_spec.sh @@ -18,7 +18,22 @@ Describe "FalkorDB switchover timeout contract" "spec", "lifecycleActions", "switchover", "timeoutSeconds" ) abort "rendered switchover timeout is #{timeout.inspect}" unless timeout == -1 - puts "definition=#{component.dig("metadata", "name")},timeout=#{timeout}" + sentinel_vars = %w[ + SENTINEL_COMPONENT_NAME + SENTINEL_USER + SENTINEL_PASSWORD + SENTINEL_POD_NAME_LIST + SENTINEL_POD_FQDN_LIST + SENTINEL_SERVICE_PORT + ] + vars = component.dig("spec", "vars").to_h { |var| [var["name"], var] } + refs = sentinel_vars.map do |name| + value_from = vars.fetch(name).fetch("valueFrom") + selector = value_from.values.find { |value| value.is_a?(Hash) } + selector&.fetch("compDef", nil) + end + abort "Sentinel references are #{refs.inspect}" unless refs == Array.new(6, "falkordb-sent-4") + puts "definition=#{component.dig("metadata", "name")},timeout=#{timeout},sentinelRefs=stable:6" ' ruby -ryaml -e ' @@ -30,12 +45,12 @@ Describe "FalkorDB switchover timeout contract" component = components.first force = migration.dig("spec", "force") abort "migration must not force the upgrade" unless force.nil? || force == false - abort "migration must preserve serviceVersion" if component.key?("serviceVersion") + abort "migration must pin serviceVersion" unless component["serviceVersion"] == "4.12.5" puts [ "migration=#{migration.dig("spec", "type")}", "component=#{component["componentName"]}", "target=#{component["componentDefinitionName"]}", - "serviceVersion=preserved", + "serviceVersion=#{component["serviceVersion"]}", "force=false" ].join(",") ' "$(chart_path)" @@ -47,8 +62,13 @@ Describe "FalkorDB switchover timeout contract" required = [ "old_pod_uids", ".items[0].spec.compDef == $target", + ".items[0].spec.serviceVersion == $service_version", "($old_uids | index($uid)) == null", ".metadata.deletionTimestamp == null", + "pod_image_contract", + "imageID", + "sentinel_env_contract", + "expected_sentinel_identity", ".name == \"KB_AGENT_ACTION\"", ".timeoutSeconds == -1", "--request-timeout=", @@ -57,16 +77,16 @@ Describe "FalkorDB switchover timeout contract" ] missing = required.reject { |contract| verifier.include?(contract) } abort "migration verifier missing: #{missing.join(", ")}" unless missing.empty? - puts "readback=component+pod-uid+ready+kbagent,bounded=true" + puts "readback=component+service-version+pod-uid+ready+images+sentinel+kbagent,bounded=true" ' "$(chart_path)" } It "publishes a new definition and an explicit existing-cluster upgrade" When call rendered_upgrade_contract The status should be success - The output should equal "definition=falkordb-4-1.2.0-alpha.1,timeout=-1 -migration=Upgrade,component=falkordb,target=falkordb-4-1.2.0-alpha.1,serviceVersion=preserved,force=false -readback=component+pod-uid+ready+kbagent,bounded=true" + The output should equal "definition=falkordb-4-1.2.0-alpha.1,timeout=-1,sentinelRefs=stable:6 +migration=Upgrade,component=falkordb,target=falkordb-4-1.2.0-alpha.1,serviceVersion=4.12.5,force=false +readback=component+service-version+pod-uid+ready+images+sentinel+kbagent,bounded=true" End End diff --git a/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh b/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh index 90d049b8dd..3f20b45a92 100644 --- a/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh +++ b/addons/falkordb/scripts-ut-spec/upgrade_switchover_timeout_spec.sh @@ -5,27 +5,96 @@ Describe "FalkorDB switchover timeout upgrade verifier" chart_dir="$(cd "${SHELLSPEC_PROJECT_ROOT:-.}" && pwd)/addons/falkordb" mock_bin_dir="$(mktemp -d)" pod_read_count_file="$mock_bin_dir/pod-read-count" + component_read_count_file="$mock_bin_dir/component-read-count" + cluster_component_read_count_file="$mock_bin_dir/cluster-component-read-count" original_path="$PATH" cat >"$mock_bin_dir/kubectl" <<'MOCK' #!/usr/bin/env bash set -euo pipefail +sentinel_env='[ + {"name":"SENTINEL_COMPONENT_NAME","value":"cluster-falkordb-sent"}, + {"name":"SENTINEL_USER","valueFrom":{"secretKeyRef":{"name":"sentinel","key":"username"}}}, + {"name":"SENTINEL_PASSWORD","valueFrom":{"secretKeyRef":{"name":"sentinel","key":"password"}}}, + {"name":"SENTINEL_POD_NAME_LIST","value":"sentinel-0,sentinel-1,sentinel-2"}, + {"name":"SENTINEL_POD_FQDN_LIST","value":"sentinel-0.headless,sentinel-1.headless,sentinel-2.headless"}, + {"name":"SENTINEL_SERVICE_PORT","value":"26379"} +]' + if [[ "$*" == *"create --dry-run=client -o json"* ]]; then if [[ "${STALL_PREFLIGHT:-0}" == "1" ]]; then sleep 5 fi cat <<'JSON' -{"kind":"OpsRequest","metadata":{"name":"upgrade","namespace":"demo"},"spec":{"clusterName":"cluster","type":"Upgrade","upgrade":{"components":[{"componentName":"falkordb","componentDefinitionName":"falkordb-4-1.2.0-alpha.1"}]}}} +{"kind":"OpsRequest","metadata":{"name":"upgrade","namespace":"demo"},"spec":{"clusterName":"cluster","type":"Upgrade","upgrade":{"components":[{"componentName":"falkordb","componentDefinitionName":"falkordb-4-1.2.0-alpha.1","serviceVersion":"4.12.5"}]}}} JSON elif [[ "$*" == *" create -f "* ]]; then printf 'opsrequest.operations.kubeblocks.io/upgrade created\n' elif [[ "$*" == *"get opsrequest"* ]]; then printf 'Succeed' +elif [[ "$*" == *"get components.apps.kubeblocks.io"* && "$*" == *"apps.kubeblocks.io/component-name=falkordb"* ]]; then + count=0 + if [[ -f "$COMPONENT_READ_COUNT_FILE" ]]; then + count="$(cat "$COMPONENT_READ_COUNT_FILE")" + fi + count=$((count + 1)) + printf '%s\n' "$count" >"$COMPONENT_READ_COUNT_FILE" + comp_def="falkordb-4-1.2.0-alpha.0" + service_version="4.12.5" + if ((count > 1)); then + comp_def="falkordb-4-1.2.0-alpha.1" + if [[ "${POST_VERSION_DRIFT:-0}" == "1" ]]; then + service_version="4.14.12" + fi + elif [[ "${CURRENT_VERSION_MISMATCH:-0}" == "1" ]]; then + service_version="4.14.12" + fi + printf '{"items":[{"metadata":{"labels":{"apps.kubeblocks.io/component-name":"falkordb"}},"spec":{"compDef":"%s","serviceVersion":"%s"}}]}\n' \ + "$comp_def" "$service_version" elif [[ "$*" == *"get components.apps.kubeblocks.io"* ]]; then - cat <<'JSON' -{"items":[{"spec":{"compDef":"falkordb-4-1.2.0-alpha.1"}}]} + count=0 + if [[ -f "$CLUSTER_COMPONENT_READ_COUNT_FILE" ]]; then + count="$(cat "$CLUSTER_COMPONENT_READ_COUNT_FILE")" + fi + count=$((count + 1)) + printf '%s\n' "$count" >"$CLUSTER_COMPONENT_READ_COUNT_FILE" + if [[ "${STANDALONE:-0}" == "1" ]]; then + cat <<'JSON' +{"items":[{"metadata":{"name":"cluster-falkordb","uid":"main-uid","labels":{"apps.kubeblocks.io/component-name":"falkordb"}},"spec":{"compDef":"falkordb-4-1.2.0-alpha.0","serviceVersion":"4.12.5"}}]} JSON + elif [[ "${SENTINEL_ABSENT:-0}" == "1" ]] || + [[ "${SENTINEL_DISAPPEARS:-0}" == "1" && "$count" -gt 1 ]]; then + cat <<'JSON' +{"items":[{"metadata":{"name":"cluster-falkordb","uid":"main-uid","labels":{"apps.kubeblocks.io/component-name":"falkordb"}},"spec":{"compDef":"falkordb-4-1.2.0-alpha.0","serviceVersion":"4.12.5"}}]} +JSON + else + deletion_timestamp=null + if [[ "${SENTINEL_TERMINATING:-0}" == "1" ]]; then + deletion_timestamp='"2026-07-31T00:00:00Z"' + fi + jq -cn --argjson deletion_timestamp "$deletion_timestamp" '{ + items: [ + { + metadata: { + name: "cluster-falkordb", + uid: "main-uid", + labels: {"apps.kubeblocks.io/component-name": "falkordb"} + }, + spec: {compDef: "falkordb-4-1.2.0-alpha.0", serviceVersion: "4.12.5"} + }, + { + metadata: { + name: "cluster-falkordb-sent", + uid: "sentinel-uid", + deletionTimestamp: $deletion_timestamp, + labels: {"apps.kubeblocks.io/component-name": "falkordb-sent"} + }, + spec: {compDef: "falkordb-sent-4-1.2.0-alpha.0", serviceVersion: "4.12.5"} + } + ] + }' + fi elif [[ "$*" == *"get pods"* ]]; then count=0 if [[ -f "$POD_READ_COUNT_FILE" ]]; then @@ -34,21 +103,93 @@ elif [[ "$*" == *"get pods"* ]]; then count=$((count + 1)) printf '%s\n' "$count" >"$POD_READ_COUNT_FILE" if ((count == 1 || ${KEEP_OLD_POD:-0} == 1)); then - cat <<'JSON' -{"items":[{"metadata":{"uid":"old-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":0}]"}]}]}}]} -JSON - elif [[ "${CONFLICTING_ACTION:-0}" == "1" ]]; then - cat <<'JSON' -{"items":[{"metadata":{"uid":"new-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":-1},{\"name\":\"switchover\",\"timeoutSeconds\":0}]"}]}]}}]} -JSON - elif [[ "${TERMINATING_POD:-0}" == "1" ]]; then - cat <<'JSON' -{"items":[{"metadata":{"uid":"new-uid","deletionTimestamp":"2026-07-31T00:00:00Z"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":-1}]"}]}]}}]} -JSON + if [[ "${STANDALONE:-0}" == "1" ]]; then + sentinel_env='[]' + fi + jq -cn \ + --argjson sentinel_env "$sentinel_env" \ + '{ + items: [{ + metadata: {name: "cluster-falkordb-0", uid: "old-uid"}, + spec: { + containers: [ + {name: "falkordb", image: "docker.io/falkordb/falkordb:v4.12.5", env: $sentinel_env}, + {name: "kbagent", image: "docker.io/apecloud/kbagent:v0", env: [ + {name: "KB_AGENT_ACTION", value: "[{\"name\":\"switchover\",\"timeoutSeconds\":0}]"} + ]} + ] + }, + status: { + conditions: [{type: "Ready", status: "True"}], + containerStatuses: [ + {name: "falkordb", image: "docker.io/falkordb/falkordb:v4.12.5", imageID: "docker-pullable://falkordb@sha256:db"}, + {name: "kbagent", image: "docker.io/apecloud/kbagent:v0", imageID: "docker-pullable://kbagent@sha256:agent"} + ] + } + }] + }' else - cat <<'JSON' -{"items":[{"metadata":{"uid":"new-uid"},"status":{"conditions":[{"type":"Ready","status":"True"}]},"spec":{"containers":[{"name":"kbagent","env":[{"name":"KB_AGENT_ACTION","value":"[{\"name\":\"switchover\",\"timeoutSeconds\":-1}]"}]}]}}]} -JSON + falkordb_image="docker.io/falkordb/falkordb:v4.12.5" + falkordb_image_id="docker-pullable://falkordb@sha256:db" + if [[ "${IMAGE_DRIFT:-0}" == "1" ]]; then + falkordb_image="docker.io/falkordb/falkordb:v4.14.12" + falkordb_image_id="docker-pullable://falkordb@sha256:new-db" + fi + if [[ "${IMAGE_ID_DRIFT:-0}" == "1" ]]; then + falkordb_image_id="docker-pullable://falkordb@sha256:retagged-db" + fi + if [[ "${MISSING_SENTINEL_ENV:-0}" == "1" ]]; then + sentinel_env='[]' + fi + if [[ "${DRIFT_SENTINEL_ENV:-0}" == "1" ]]; then + sentinel_env="$(jq -c ' + map( + if .name == "SENTINEL_COMPONENT_NAME" then + .value = "cluster-other-sentinel" + else + . + end + ) + ' <<<"$sentinel_env")" + fi + switchover_actions='[{"name":"switchover","timeoutSeconds":-1}]' + if [[ "${CONFLICTING_ACTION:-0}" == "1" ]]; then + switchover_actions='[{"name":"switchover","timeoutSeconds":-1},{"name":"switchover","timeoutSeconds":0}]' + fi + deletion_timestamp=null + if [[ "${TERMINATING_POD:-0}" == "1" ]]; then + deletion_timestamp='"2026-07-31T00:00:00Z"' + fi + jq -cn \ + --arg falkordb_image "$falkordb_image" \ + --arg falkordb_image_id "$falkordb_image_id" \ + --arg actions "$switchover_actions" \ + --argjson deletion_timestamp "$deletion_timestamp" \ + --argjson sentinel_env "$sentinel_env" \ + '{ + items: [{ + metadata: { + name: "cluster-falkordb-0", + uid: "new-uid", + deletionTimestamp: $deletion_timestamp + }, + spec: { + containers: [ + {name: "falkordb", image: $falkordb_image, env: $sentinel_env}, + {name: "kbagent", image: "docker.io/apecloud/kbagent:v0", env: [ + {name: "KB_AGENT_ACTION", value: $actions} + ]} + ] + }, + status: { + conditions: [{type: "Ready", status: "True"}], + containerStatuses: [ + {name: "falkordb", image: $falkordb_image, imageID: $falkordb_image_id}, + {name: "kbagent", image: "docker.io/apecloud/kbagent:v0", imageID: "docker-pullable://kbagent@sha256:agent"} + ] + } + }] + }' fi else printf 'unexpected kubectl call: %s\n' "$*" >&2 @@ -58,6 +199,8 @@ MOCK chmod +x "$mock_bin_dir/kubectl" export PATH="$mock_bin_dir:$PATH" export POD_READ_COUNT_FILE="$pod_read_count_file" + export COMPONENT_READ_COUNT_FILE="$component_read_count_file" + export CLUSTER_COMPONENT_READ_COUNT_FILE="$cluster_component_read_count_file" export UPGRADE_VERIFY_TIMEOUT_SECONDS=2 export UPGRADE_VERIFY_POLL_SECONDS=1 export UPGRADE_VERIFY_API_TIMEOUT_SECONDS=1 @@ -66,8 +209,13 @@ MOCK cleanup_mock_kubectl() { PATH="$original_path" rm -rf "$mock_bin_dir" - unset chart_dir mock_bin_dir pod_read_count_file original_path - unset POD_READ_COUNT_FILE KEEP_OLD_POD CONFLICTING_ACTION TERMINATING_POD STALL_PREFLIGHT + unset chart_dir mock_bin_dir pod_read_count_file component_read_count_file + unset cluster_component_read_count_file original_path + unset POD_READ_COUNT_FILE COMPONENT_READ_COUNT_FILE CLUSTER_COMPONENT_READ_COUNT_FILE + unset KEEP_OLD_POD CONFLICTING_ACTION TERMINATING_POD STALL_PREFLIGHT + unset CURRENT_VERSION_MISMATCH POST_VERSION_DRIFT IMAGE_DRIFT IMAGE_ID_DRIFT + unset MISSING_SENTINEL_ENV STANDALONE + unset SENTINEL_ABSENT SENTINEL_TERMINATING SENTINEL_DISAPPEARS DRIFT_SENTINEL_ENV unset UPGRADE_VERIFY_TIMEOUT_SECONDS UPGRADE_VERIFY_POLL_SECONDS unset UPGRADE_VERIFY_API_TIMEOUT_SECONDS FALKORDB_UPGRADE_VERIFY_SUPERVISED } @@ -75,12 +223,103 @@ MOCK BeforeEach "setup_mock_kubectl" AfterEach "cleanup_mock_kubectl" - It "accepts only a new ready Pod with the target kbagent timeout" + It "accepts a replication replacement with stable version, images, Sentinel vars, and timeout" + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be success + The stdout should include "Upgrade verified" + The stdout should include "serviceVersion=4.12.5" + The stdout should include "recreatedPods=1 images=unchanged" + End + + It "accepts a standalone replacement without Sentinel vars" + export STANDALONE=1 + export MISSING_SENTINEL_ENV=1 When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ "$chart_dir/examples/upgrade-switchover-timeout.yaml" The status should be success The stdout should include "Upgrade verified" - The stdout should include "recreatedPods=1" + End + + It "rejects a manifest serviceVersion that differs from the live Component before creation" + export CURRENT_VERSION_MISMATCH=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should not include "created" + The stderr should include "does not match live Component serviceVersion 4.14.12" + End + + It "fails closed when the replacement Component changes serviceVersion" + export POST_VERSION_DRIFT=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "fails closed when replacement Pod image names or IDs drift" + export IMAGE_DRIFT=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "fails closed when only a replacement Pod image ID drifts" + export IMAGE_ID_DRIFT=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "fails closed when a replication replacement loses Sentinel variables" + export MISSING_SENTINEL_ENV=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "rejects replication preflight when old Pods require an absent Sentinel sibling" + export SENTINEL_ABSENT=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should not include "created" + The stderr should include "existing Pods require Sentinel" + End + + It "rejects replication preflight when the Sentinel sibling is terminating" + export SENTINEL_TERMINATING=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should not include "created" + The stderr should include "expected at most one non-terminating FalkorDB Sentinel Component" + End + + It "fails closed when the Sentinel sibling disappears after creation" + export SENTINEL_DISAPPEARS=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" + End + + It "fails closed when replacement Sentinel values drift" + export DRIFT_SENTINEL_ENV=1 + When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ + "$chart_dir/examples/upgrade-switchover-timeout.yaml" + The status should be failure + The stdout should include "created" + The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" End It "fails closed when the old Pod UID remains" @@ -88,7 +327,7 @@ MOCK When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ "$chart_dir/examples/upgrade-switchover-timeout.yaml" The status should be failure - The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stdout should include "created" The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" End @@ -97,7 +336,7 @@ MOCK When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ "$chart_dir/examples/upgrade-switchover-timeout.yaml" The status should be failure - The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stdout should include "created" The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" End @@ -106,7 +345,7 @@ MOCK When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ "$chart_dir/examples/upgrade-switchover-timeout.yaml" The status should be failure - The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stdout should include "created" The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" End @@ -117,7 +356,7 @@ MOCK When run command bash "$chart_dir/examples/upgrade-switchover-timeout.sh" \ "$chart_dir/examples/upgrade-switchover-timeout.yaml" The status should be failure - The stdout should include "opsrequest.operations.kubeblocks.io/upgrade created" + The stdout should include "created" The stderr should include "timed out verifying ComponentDefinition and recreated kbagent Pods" End diff --git a/addons/falkordb/templates/cmpd-falkordb.yaml b/addons/falkordb/templates/cmpd-falkordb.yaml index a9966d2369..0c321dd463 100644 --- a/addons/falkordb/templates/cmpd-falkordb.yaml +++ b/addons/falkordb/templates/cmpd-falkordb.yaml @@ -235,7 +235,7 @@ spec: - name: SENTINEL_COMPONENT_NAME valueFrom: componentVarRef: - compDef: {{ printf "%s-%s" .sentinelComponentDef $.Chart.Version }} + compDef: {{ .sentinelComponentDef }} optional: true componentName: Required ## the username of falkordb sentinel for falkordb connection, it is optional @@ -243,7 +243,7 @@ spec: valueFrom: credentialVarRef: ## reference the falkordb-sent component definition name - compDef: {{ printf "%s-%s" .sentinelComponentDef $.Chart.Version }} + compDef: {{ .sentinelComponentDef }} name: default optional: true username: Required @@ -259,27 +259,27 @@ spec: valueFrom: credentialVarRef: ## reference the falkordb-sent component definition name - compDef: {{ printf "%s-%s" .sentinelComponentDef $.Chart.Version }} + compDef: {{ .sentinelComponentDef }} name: default optional: true password: Required - name: SENTINEL_POD_NAME_LIST valueFrom: componentVarRef: - compDef: {{ printf "%s-%s" .sentinelComponentDef $.Chart.Version }} + compDef: {{ .sentinelComponentDef }} optional: true podNames: Required - name: SENTINEL_POD_FQDN_LIST valueFrom: componentVarRef: - compDef: {{ printf "%s-%s" .sentinelComponentDef $.Chart.Version }} + compDef: {{ .sentinelComponentDef }} optional: true podFQDNs: Required ## the falkordb sentinel server port, if falkordb sentinel is in host network mode, the port will be allocated automatically by KubeBlocks, if not, the default port is 26379 - name: SENTINEL_SERVICE_PORT valueFrom: hostNetworkVarRef: - compDef: {{ printf "%s-%s" .sentinelComponentDef $.Chart.Version }} + compDef: {{ .sentinelComponentDef }} optional: true container: name: falkordb-sent