Skip to content

Commit 4f4c7b7

Browse files
authored
Merge pull request #2330 from AndrewSirenko/scaleResizer
Add expand-and-modify test type to ebs-scale-test
2 parents b884caf + ef6dcd4 commit 4f4c7b7

File tree

7 files changed

+181
-6
lines changed

7 files changed

+181
-6
lines changed

hack/ebs-scale-test/README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,8 @@ Set the `CLUSTER_TYPE` and `TEST_TYPE` environment variables to set up and run d
6464
- 'pre-allocated': Additional worker nodes are created during cluster setup. By default, we pre-allocate 1 `m7a.48xlarge` EC2 instance for every 100 StatefulSet replicas.
6565

6666
- `TEST_TYPE` dictates what type of scalability test we want to run. Options include:
67-
- 'scale-sts': Scales a StatefulSet to `$REPLICAS`. Waits for all pods to be ready. Delete Sts. Waits for all PVs to be deleted. Exercises the complete dynamic provisioning lifecycle for block volumes.
67+
- 'scale-sts': Scales a StatefulSet to `$REPLICAS`. Waits for all pods to be ready. Delete Sts. Waits for all PVs to be deleted. Exercises the complete dynamic provisioning lifecycle for block volumes.
68+
- 'expand-and-modify': Creates `$REPLICAS` block volumes. Patches PVC capacity and VACName at rate of 5 PVCs per second. Ensures PVCs are expanded and modified before deleting them. Exercises ControllerExpandVolume & ControllerModifyVolume. Set `MODIFY_ONLY` or `EXPAND_ONLY` to 'true' to test solely volume modification/expansion.
6869

6970
You can mix and match `CLUSTER_TYPE` and `TEST_TYPE`.
7071

hack/ebs-scale-test/helpers/scale-test/collect-and-export-metrics.sh

+6
Original file line numberDiff line numberDiff line change
@@ -41,9 +41,13 @@ collect-and-export-metrics() {
4141
collect_metrics() {
4242
echo "Port-forwarding ebs-csi-controller containers"
4343
kubectl port-forward "$CONTROLLER_POD_NAME" 3301:3301 -n kube-system &
44+
PID_3301=$!
4445
kubectl port-forward "$CONTROLLER_POD_NAME" 8081:8081 -n kube-system &
46+
PID_8081=$!
4547
kubectl port-forward "$CONTROLLER_POD_NAME" 8082:8082 -n kube-system &
48+
PID_8082=$!
4649
kubectl port-forward "$CONTROLLER_POD_NAME" 8084:8084 -n kube-system &
50+
PID_8084=$!
4751

4852
echo "Collecting metrics"
4953
for port in 3301 8081 8082 8084; do
@@ -58,6 +62,8 @@ collect_metrics() {
5862
sleep 20
5963
echo "WARNING: Could not collect metrics from port ${port}. Something may be wrong in cluster."
6064
done
65+
# Stop forwarding ports after metrics collected.
66+
kill $PID_3301 $PID_8081 $PID_8082 $PID_8084
6167
}
6268

6369
clean_metrics() {
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
#!/bin/bash
2+
# Copyright 2025 The Kubernetes Authors.
3+
#
4+
# Licensed under the Apache License, Version 2.0 (the "License");
5+
# you may not use this file except in compliance with the License.
6+
# You may obtain a copy of the License at
7+
#
8+
# http://www.apache.org/licenses/LICENSE-2.0
9+
#
10+
# Unless required by applicable law or agreed to in writing, software
11+
# distributed under the License is distributed on an "AS IS" BASIS,
12+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
# See the License for the specific language governing permissions and
14+
# limitations under the License.
15+
16+
### Helper script for running EBS-backed PVC expansion and modification test
17+
18+
# We expect this helper script is sourced from hack/ebs-scale-test
19+
path_to_resize_and_modify_test_dir="${BASE_DIR}/helpers/scale-test/expand-and-modify-test"
20+
21+
export EXPAND_ONLY=${EXPAND_ONLY:="false"}
22+
export MODIFY_ONLY=${MODIFY_ONLY:="false"}
23+
24+
expand_and_modify_test() {
25+
manifest_path="$path_to_resize_and_modify_test_dir/expand-and-modify.yaml"
26+
export_manifest_path="$EXPORT_DIR/expand-and-modify.yaml"
27+
28+
echo "Applying $manifest_path. Exported to $export_manifest_path"
29+
gomplate -f "$manifest_path" -o "$export_manifest_path"
30+
kubectl apply -f "$export_manifest_path"
31+
32+
# Cleanup K8s resources upon script interruption
33+
trap 'echo "Test interrupted! Deleting test resources to prevent leak"; kubectl delete -f $export_manifest_path' EXIT
34+
35+
echo "Waiting for all PVCs to be dynamically provisioned"
36+
wait_for_pvcs_to_bind
37+
38+
case "$EXPAND_ONLY-$MODIFY_ONLY" in
39+
*false-false*) patches='[{"op": "replace", "path": "/spec/volumeAttributesClassName", "value": "ebs-scale-test-expand-and-modify"},{"op": "replace", "path": "/spec/resources/requests/storage", "value": "2Gi"}]' ;;
40+
*false-true*) patches='[{"op": "replace", "path": "/spec/volumeAttributesClassName", "value": "ebs-scale-test-expand-and-modify"}]' ;;
41+
*true-false* | *delete*) patches='[{"op": "replace", "path": "/spec/resources/requests/storage", "value": "2Gi"}]' ;;
42+
*) echo "Environment variables EXPAND_ONLY ('$EXPAND_ONLY') and MODIFY_ONLY ('$MODIFY_ONLY') are not properly set to either 'true' or 'false'" ;;
43+
esac
44+
45+
echo "Patching PVCs with $patches"
46+
kubectl get pvc -o name | sed -e 's/.*\///g' | xargs -P 5 -I {} kubectl patch pvc {} --type=json -p="$patches"
47+
48+
echo "Waiting until volumes modified and/or expanded"
49+
ensure_volumes_modified
50+
51+
echo "Deleting resources"
52+
kubectl delete -f "$export_manifest_path"
53+
54+
echo "Waiting for all PVs to be deleted"
55+
wait_for_pvs_to_delete
56+
57+
trap - EXIT
58+
}
59+
60+
ensure_volumes_modified() {
61+
if [[ "$EXPAND_ONLY" == "false" ]]; then
62+
while true; do
63+
modified_volumes_count=$(kubectl get pvc -o json | jq '.items | map(select(.status.currentVolumeAttributesClassName == "ebs-scale-test-expand-and-modify")) | length')
64+
echo "$modified_volumes_count/$REPLICAS volumes modified"
65+
if [[ "$modified_volumes_count" == "$REPLICAS" ]]; then
66+
echo "All volumes modified"
67+
break
68+
fi
69+
sleep 5
70+
done
71+
fi
72+
73+
if [[ "$MODIFY_ONLY" == "false" ]]; then
74+
while true; do
75+
expanded_volumes_count=$(kubectl get pvc -o json | jq '.items | map(select(.status.capacity.storage == "2Gi")) | length')
76+
echo "$expanded_volumes_count/$REPLICAS volumes expanded"
77+
if [[ "$expanded_volumes_count" == "$REPLICAS" ]]; then
78+
echo "All volumes expanded"
79+
break
80+
fi
81+
sleep 5
82+
done
83+
fi
84+
}
85+
86+
wait_for_pvcs_to_bind() {
87+
while true; do
88+
bound_pvc_count=$(kubectl get pvc -o json | jq '.items | map(select(.status.phase == "Bound")) | length')
89+
if [[ "$bound_pvc_count" -ge "$REPLICAS" ]]; then
90+
echo "All PVCs bound, proceeding..."
91+
break
92+
else
93+
echo "Only $bound_pvc_count PVCs are bound, waiting for a total of $REPLICAS..."
94+
sleep 5
95+
fi
96+
done
97+
}
98+
99+
(return 0 2>/dev/null) || (
100+
echo "This script is not meant to be run directly, only sourced as a helper!"
101+
exit 1
102+
)
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
# Copyright 2025 The Kubernetes Authors.
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
---
16+
apiVersion: storage.k8s.io/v1
17+
kind: StorageClass
18+
metadata:
19+
name: ebs-scale-test-expand-and-modify
20+
provisioner: ebs.csi.aws.com
21+
reclaimPolicy: Delete
22+
allowVolumeExpansion: true
23+
# We create immediately bounded volumes so that we do not have to attach/detach volumes in expand-and-modify scale test.
24+
volumeBindingMode: Immediate
25+
parameters:
26+
type: gp2
27+
tagSpecification_1: "ebs-scale-test={{ .Env.SCALABILITY_TEST_RUN_NAME }}"
28+
---
29+
apiVersion: storage.k8s.io/v1beta1
30+
kind: VolumeAttributesClass
31+
driverName: ebs.csi.aws.com
32+
metadata:
33+
name: ebs-scale-test-expand-and-modify
34+
parameters:
35+
type: gp3
36+
---
37+
{{- range $index, $value := seq 1 .Env.REPLICAS }}
38+
apiVersion: v1
39+
kind: PersistentVolumeClaim
40+
metadata:
41+
name: ebs-claim-{{ $index }}
42+
spec:
43+
accessModes:
44+
- ReadWriteOnce
45+
storageClassName: ebs-scale-test-expand-and-modify
46+
resources:
47+
requests:
48+
storage: 1Gi
49+
# We use block mode so that no filesystem expansion occurs (via NodeExpandVolume)
50+
volumeMode: "Block"
51+
---
52+
{{- end }}

hack/ebs-scale-test/helpers/scale-test/pre_test_validation.sh

+3-2
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,8 @@ pre_test_validation() {
2525
exit 1
2626
fi
2727

28-
echo "Updating kubeconfig and restarting ebs-csi-controller pod"
28+
echo "Updating kubeconfig and restarting ebs-csi-controller Deployment"
2929
aws eks update-kubeconfig --name "$CLUSTER_NAME"
30-
kubectl delete pod -n kube-system -l app=ebs-csi-controller
30+
kubectl rollout restart deployment/ebs-csi-controller -n kube-system
31+
kubectl rollout status deployment/ebs-csi-controller -n kube-system --timeout=30s
3132
}

hack/ebs-scale-test/helpers/scale-test/scale-sts-test/scale-sts.sh

+6-1
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ sts_scale_test() {
2626
gomplate -f "$manifest_path" -o "$export_manifest_path"
2727
kubectl apply -f "$export_manifest_path"
2828

29+
# Cleanup K8s resources upon script interruption
30+
trap 'echo "Test interrupted! Deleting test resources to prevent leak"; kubectl delete -f $export_manifest_path' EXIT
31+
2932
echo "Scaling StatefulSet $REPLICAS replicas"
3033
kubectl scale sts --replicas "$REPLICAS" ebs-scale-test
3134
kubectl rollout status statefulset ebs-scale-test
@@ -35,11 +38,13 @@ sts_scale_test() {
3538

3639
echo "Waiting for all PVs to be deleted"
3740
wait_for_pvs_to_delete
41+
42+
trap - EXIT
3843
}
3944

4045
wait_for_pvs_to_delete() {
4146
while true; do
42-
pv_count=$(kubectl get pv --no-headers | wc -l)
47+
pv_count=$(kubectl get pv -o json | jq '.items | length')
4348
if [ "$pv_count" -eq 0 ]; then
4449
echo "No PVs exist in the cluster, proceeding..."
4550
break

hack/ebs-scale-test/scale-test

+10-2
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ source "${BASE_DIR}/helpers/scale-test/collect-and-export-metrics.sh"
5656
source "${BASE_DIR}/helpers/scale-test/pre_test_validation.sh"
5757

5858
source "${BASE_DIR}/helpers/scale-test/scale-sts-test/scale-sts.sh"
59+
source "${BASE_DIR}/helpers/scale-test/expand-and-modify-test/expand-and-modify.sh"
5960

6061
usage() {
6162
echo "Usage: $0 [base-cmd]"
@@ -64,7 +65,7 @@ usage() {
6465
}
6566

6667
check_dependencies_helper() {
67-
local readonly dependencies=("kubectl" "aws" "eksctl" "gomplate")
68+
local readonly dependencies=("kubectl" "aws" "eksctl" "gomplate" "jq")
6869

6970
for cmd in "${dependencies[@]}"; do
7071
if ! command -v "${cmd}" &>/dev/null; then
@@ -86,7 +87,14 @@ setup_scale() {
8687
run_scale() {
8788
pre_test_validation
8889

89-
sts_scale_test
90+
case "$TEST_TYPE" in
91+
*scale-sts*) sts_scale_test ;;
92+
*expand-and-modify*) expand_and_modify_test ;;
93+
*)
94+
echo "Invalid TEST_TYPE '$TEST_TYPE'. Please set to 'scale-sts' or 'expand-and-modify'"
95+
exit 1
96+
;;
97+
esac
9098

9199
collect-and-export-metrics
92100
}

0 commit comments

Comments
 (0)