Skip to content

fix(etcd): fail role probe on status errors #3267

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
weicao wants to merge 1 commit into
base: martin/etcd-memberjoin-idempotency
Choose a base branch
from
+237 −112
Draft

fix(etcd): fail role probe on status errors #3267

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
285 changes: 184 additions & 101 deletions addons/etcd/scripts-ut-spec/roleprobe_spec.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,132 +1,215 @@
# shellcheck shell=bash
# shellcheck disable=SC2034
# shellcheck disable=SC2034,SC2329

# validate_shell_type_and_version defined in shellspec/spec_helper.sh used to validate the expected shell type and version this script needs to run.
if ! validate_shell_type_and_version "bash" 4 &>/dev/null; then
echo "roleprobe_spec.sh skip cases because dependency bash version 4 or higher is not installed."
exit 0
fi

source ./utils.sh

# The unit test needs to rely on the common library functions defined in kblib.
# Therefore, we first dynamically generate the required common library files from the kblib library chart.
common_library_file="./common.sh"
generate_common_library $common_library_file

Describe "Etcd Role Probe Script Tests"
# load the scripts to be tested and dependencies
Include $common_library_file
Include ../scripts/roleprobe.sh

setup() {
TEST_DIR=$(mktemp -d)
export TEST_DIR
export ROLEPROBE_ENTRYPOINT_DIR="$TEST_DIR/scripts"
export ROLEPROBE_ETCDCTL_RC=0
export ROLEPROBE_ETCDCTL_STDOUT=
export ROLEPROBE_ETCDCTL_STDERR=

mkdir -p "$ROLEPROBE_ENTRYPOINT_DIR"
cp ../scripts/roleprobe.sh "$ROLEPROBE_ENTRYPOINT_DIR/roleprobe.sh"
cat > "$ROLEPROBE_ENTRYPOINT_DIR/common.sh" <<'COMMON'
setup_shellspec() { :; }
load_common_library() { :; }
exec_etcdctl() {
if [ "$*" != "127.0.0.1:2379 endpoint status -w fields --command-timeout=300ms --dial-timeout=100ms" ]; then
printf 'unexpected etcdctl arguments: %s\n' "$*" >&2
return 64
fi
printf '%s' "${ROLEPROBE_ETCDCTL_STDOUT:-}"
printf '%s' "${ROLEPROBE_ETCDCTL_STDERR:-}" >&2
return "${ROLEPROBE_ETCDCTL_RC:-0}"
}
COMMON

init() {
# set ut_mode to true to hack control flow in the script
ut_mode="true"

# Mock exec_etcdctl function
exec_etcdctl() {
local endpoint="$1"
shift
case "$*" in
*"endpoint status -w fields"*)
echo '"MemberID" : 1002'
echo '"Leader" : 1002'
echo '"IsLearner" : false'
return 0
;;
*)
echo "MOCK: exec_etcdctl $endpoint $*"
return 0
;;
esac
}

# Define get_etcd_role function based on real script logic
get_etcd_role() {
local status member_id leader_id is_learner

if ! status=$(exec_etcdctl 127.0.0.1:2379 endpoint status -w fields --command-timeout=300ms --dial-timeout=100ms); then
echo "ERROR: Failed to get endpoint status" >&2
return 1
fi

member_id=$(echo "$status" | grep -o '"MemberID" : [0-9]*' | awk '{print $3}')
leader_id=$(echo "$status" | grep -o '"Leader" : [0-9]*' | awk '{print $3}')
is_learner=$(echo "$status" | grep -o '"IsLearner" : [a-z]*' | awk '{print $3}')

# Check if required fields are present
if [ -z "$member_id" ] || [ -z "$leader_id" ]; then
echo "follower" # Default to follower when fields are missing
return 0
fi

if [ "$member_id" = "$leader_id" ]; then
echo "leader"
elif [ "$is_learner" = "true" ]; then
echo "learner"
else
echo "follower"
fi
printf '%s' "${ROLEPROBE_ETCDCTL_STDOUT:-}"
printf '%s' "${ROLEPROBE_ETCDCTL_STDERR:-}" >&2
return "${ROLEPROBE_ETCDCTL_RC:-0}"
}
}
BeforeAll "init"
BeforeEach "setup"

cleanup() {
rm -f $common_library_file
unset ut_mode
unset -f exec_etcdctl get_etcd_role
rm -rf "$TEST_DIR"
unset TEST_DIR ROLEPROBE_ENTRYPOINT_DIR ROLEPROBE_ETCDCTL_RC
unset ROLEPROBE_ETCDCTL_STDOUT ROLEPROBE_ETCDCTL_STDERR
unset -f exec_etcdctl
}
AfterEach "cleanup"

endpoint_status() {
printf '"MemberID" : %s\n"Leader" : %s\n"IsLearner" : %s\n' "$1" "$2" "$3"
}

run_roleprobe_entrypoint() {
cmp -s ../scripts/roleprobe.sh "$ROLEPROBE_ENTRYPOINT_DIR/roleprobe.sh" ||
return 99
bash "$ROLEPROBE_ENTRYPOINT_DIR/roleprobe.sh"
}

assert_role_failure() {
ROLEPROBE_ETCDCTL_STDOUT="$1"
export ROLEPROBE_ETCDCTL_STDOUT
roleprobe_main
}
AfterAll 'cleanup'

Describe "get_etcd_role() function"
It "detects leader role correctly"
Describe "production get_etcd_role()"
It "detects an official-shape leader"
ROLEPROBE_ETCDCTL_STDOUT=$'"Endpoint" : "127.0.0.1:2379"\n"ClusterID" : 9\n'
ROLEPROBE_ETCDCTL_STDOUT+="$(endpoint_status 1002 1002 false)"
When call get_etcd_role
The status should be success
The output should equal "leader"
End

It "detects follower role correctly"
# Override exec_etcdctl to return follower status
exec_etcdctl() {
case "$*" in
*"endpoint status -w fields"*)
echo '"MemberID" : 1001'
echo '"Leader" : 1002'
echo '"IsLearner" : false'
return 0
;;
esac
}
The output should eq "leader"
End

It "detects an official-shape follower"
ROLEPROBE_ETCDCTL_STDOUT=$(endpoint_status 1001 1002 false)
When call get_etcd_role
The status should be success
The output should equal "follower"
End

It "detects learner role correctly"
# Override exec_etcdctl to return learner status
exec_etcdctl() {
case "$*" in
*"endpoint status -w fields"*)
echo '"MemberID" : 1003'
echo '"Leader" : 1002'
echo '"IsLearner" : true'
return 0
;;
esac
}
The output should eq "follower"
End

It "detects an official-shape learner"
ROLEPROBE_ETCDCTL_STDOUT=$(endpoint_status 1003 1002 true)
When call get_etcd_role
The status should be success
The output should equal "learner"
The output should eq "learner"
End

It "handles etcdctl failure"
# Override exec_etcdctl to fail
exec_etcdctl() { return 1; }

It "propagates an etcdctl failure"
ROLEPROBE_ETCDCTL_RC=23
ROLEPROBE_ETCDCTL_STDERR="controlled etcdctl failure"
When call get_etcd_role
The status should be failure
The output should eq ""
The stderr should include "controlled etcdctl failure"
The stderr should include "Failed to get endpoint status"
End

It "rejects a missing MemberID"
When call assert_role_failure $'"Leader" : 1002\n"IsLearner" : false'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects a missing Leader"
When call assert_role_failure $'"MemberID" : 1002\n"IsLearner" : false'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects a missing IsLearner"
When call assert_role_failure $'"MemberID" : 1002\n"Leader" : 1002'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects a MemberID suffix"
When call assert_role_failure "$(endpoint_status 1002garbage 1002 false)"
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects a Leader suffix"
When call assert_role_failure "$(endpoint_status 1002 1002garbage false)"
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects an invalid learner boolean"
When call assert_role_failure "$(endpoint_status 1002 1002 falsegarbage)"
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects duplicate MemberID fields"
When call assert_role_failure $'"MemberID" : 1002\n"MemberID" : 1003\n"Leader" : 1002\n"IsLearner" : false'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects a repeated identical Leader field"
When call assert_role_failure $'"MemberID" : 1002\n"Leader" : 1002\n"Leader" : 1002\n"IsLearner" : false'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects conflicting duplicate Leader fields"
When call assert_role_failure $'"MemberID" : 1002\n"Leader" : 1002\n"Leader" : 1003\n"IsLearner" : false'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End

It "rejects conflicting duplicate IsLearner fields"
When call assert_role_failure $'"MemberID" : 1002\n"Leader" : 1002\n"IsLearner" : false\n"IsLearner" : true'
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End
End

Describe "production roleprobe entrypoint"
It "emits the classified role"
ROLEPROBE_ETCDCTL_STDOUT=$(endpoint_status 1002 1002 false)
When call run_roleprobe_entrypoint
The status should be success
The output should eq "leader"
The stderr should eq ""
End

It "emits follower from the official field shape"
ROLEPROBE_ETCDCTL_STDOUT=$(endpoint_status 1001 1002 false)
When call run_roleprobe_entrypoint
The status should be success
The output should eq "follower"
The stderr should eq ""
End

It "emits learner from the official field shape"
ROLEPROBE_ETCDCTL_STDOUT=$(endpoint_status 1003 1002 true)
When call run_roleprobe_entrypoint
The status should be success
The output should eq "learner"
The stderr should eq ""
End

It "propagates etcdctl rc and keeps stdout empty"
ROLEPROBE_ETCDCTL_RC=23
ROLEPROBE_ETCDCTL_STDERR="controlled entrypoint failure"
When call run_roleprobe_entrypoint
The status should be failure
The output should eq ""
The stderr should include "controlled entrypoint failure"
The stderr should include "Failed to get endpoint status"
End

It "fails closed on malformed rc0 status"
ROLEPROBE_ETCDCTL_STDOUT=$'"MemberID" : 1002garbage\n"Leader" : 1002\n"IsLearner" : false'
When call run_roleprobe_entrypoint
The status should be failure
The output should eq ""
The stderr should include "Failed to extract role fields"
End
End
End
End
64 changes: 53 additions & 11 deletions addons/etcd/scripts/roleprobe.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,41 @@
#!/bin/bash

# shellcheck disable=SC1091
. "/scripts/common.sh"
# shellcheck disable=SC1090,SC1091
roleprobe_script_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
. "${roleprobe_script_dir}/common.sh"

parse_role_field() {
local status="$1"
local field_name="$2"
local value_type="$3"

printf '%s\n' "$status" | awk -v field_name="$field_name" -v value_type="$value_type" '
BEGIN {
key = "^[[:space:]]*\"" field_name "\"[[:space:]]*:"
if (value_type == "id") {
value = "[0-9]+"
} else {
value = "(true|false)"
}
valid = key "[[:space:]]*" value "[[:space:]]*$"
}
$0 ~ key {
seen++
if ($0 ~ valid) {
parsed = $0
sub(key "[[:space:]]*", "", parsed)
sub("[[:space:]]*$", "", parsed)
valid_count++
}
}
END {
if (seen != 1 || valid_count != 1) {
exit 1
}
print parsed
}
'
}

get_etcd_role() {
local status member_id leader_id is_learner
Expand All @@ -10,9 +44,12 @@ get_etcd_role() {
return 1
fi

member_id=$(echo "$status" | grep -o '"MemberID" : [0-9]*' | awk '{print $3}')
leader_id=$(echo "$status" | grep -o '"Leader" : [0-9]*' | awk '{print $3}')
is_learner=$(echo "$status" | grep -o '"IsLearner" : [a-z]*' | awk '{print $3}')
if ! member_id=$(parse_role_field "$status" "MemberID" "id") ||
! leader_id=$(parse_role_field "$status" "Leader" "id") ||
! is_learner=$(parse_role_field "$status" "IsLearner" "bool"); then
echo "ERROR: Failed to extract role fields from endpoint status" >&2
return 1
fi

if [ "$member_id" = "$leader_id" ]; then
echo "leader"
Expand All @@ -23,10 +60,15 @@ get_etcd_role() {
fi
}

# Shellspec magic
setup_shellspec
roleprobe_main() {
local etcd_role
if ! etcd_role=$(get_etcd_role); then
return 1
fi
printf '%s' "$etcd_role"
}

# main
load_common_library
etcd_role=$(get_etcd_role)
echo -n "$etcd_role"
if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then
load_common_library
roleprobe_main "$@"
fi
Loading