|
| 1 | +#!/bin/bash |
| 2 | +# Copyright 2025 The Kubernetes Authors. |
| 3 | +# |
| 4 | +# Licensed under the Apache License, Version 2.0 (the "License"); |
| 5 | +# you may not use this file except in compliance with the License. |
| 6 | +# You may obtain a copy of the License at |
| 7 | +# |
| 8 | +# http://www.apache.org/licenses/LICENSE-2.0 |
| 9 | +# |
| 10 | +# Unless required by applicable law or agreed to in writing, software |
| 11 | +# distributed under the License is distributed on an "AS IS" BASIS, |
| 12 | +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 13 | +# See the License for the specific language governing permissions and |
| 14 | +# limitations under the License. |
| 15 | + |
| 16 | +### Helper script for exporting EBS CSI Driver metrics to S3 bucket |
| 17 | + |
| 18 | +set -euo pipefail |
| 19 | + |
| 20 | +collect-and-export-metrics() { |
| 21 | + export CONTROLLER_POD_NAME |
| 22 | + CONTROLLER_POD_NAME=$(kubectl get pod -n kube-system -l app=ebs-csi-controller -o jsonpath='{.items[0].metadata.name}') |
| 23 | + export METRICS_FILEPATH="$EXPORT_DIR/metrics.txt" |
| 24 | + |
| 25 | + collect_metrics |
| 26 | + clean_metrics |
| 27 | + |
| 28 | + echo "Collecting ebs-plugin logs" |
| 29 | + kubectl logs "$CONTROLLER_POD_NAME" -n kube-system >"$EXPORT_DIR/ebs-plugin-logs.txt" |
| 30 | + |
| 31 | + echo "Collecting ebs-csi-controller Deployment and ebs-csi-node Daemonset yaml" |
| 32 | + kubectl get deployment ebs-csi-controller -n kube-system -o yaml >"$EXPORT_DIR/ebs-csi-controller.yaml" |
| 33 | + kubectl get daemonset ebs-csi-node -n kube-system -o yaml >"$EXPORT_DIR/ebs-csi-node.yaml" |
| 34 | + |
| 35 | + echo "Exporting everything in $EXPORT_DIR to S3 bucket s3://$S3_BUCKET/$SCALABILITY_TEST_RUN_NAME" |
| 36 | + |
| 37 | + aws s3 sync "$EXPORT_DIR" "s3://$S3_BUCKET/$SCALABILITY_TEST_RUN_NAME" |
| 38 | + echo "Metrics exported to s3://$S3_BUCKET/$SCALABILITY_TEST_RUN_NAME/" |
| 39 | +} |
| 40 | + |
| 41 | +collect_metrics() { |
| 42 | + echo "Port-forwarding ebs-csi-controller containers" |
| 43 | + kubectl port-forward "$CONTROLLER_POD_NAME" 3301:3301 -n kube-system & |
| 44 | + kubectl port-forward "$CONTROLLER_POD_NAME" 8081:8081 -n kube-system & |
| 45 | + kubectl port-forward "$CONTROLLER_POD_NAME" 8082:8082 -n kube-system & |
| 46 | + kubectl port-forward "$CONTROLLER_POD_NAME" 8084:8084 -n kube-system & |
| 47 | + |
| 48 | + echo "Collecting metrics" |
| 49 | + for port in 3301 8081 8082 8084; do |
| 50 | + curl "http://localhost:${port}/metrics" >>"$METRICS_FILEPATH" && continue |
| 51 | + echo "Failed to collect metrics from port ${port}, retrying after 5s..." |
| 52 | + sleep 5 |
| 53 | + curl "http://localhost:${port}/metrics" >>"$METRICS_FILEPATH" && continue |
| 54 | + echo "Failed to collect metrics from port ${port} AGAIN, retrying after 10s..." |
| 55 | + sleep 10 |
| 56 | + curl "http://localhost:${port}/metrics" >>"$METRICS_FILEPATH" && continue |
| 57 | + echo "Failed to collect metrics from port ${port} THRICE, retrying one more time after 20s..." |
| 58 | + sleep 20 |
| 59 | + echo "WARNING: Could not collect metrics from port ${port}. Something may be wrong in cluster." |
| 60 | + done |
| 61 | +} |
| 62 | + |
| 63 | +clean_metrics() { |
| 64 | + echo "Generating clean version of exported data at $EXPORT_DIR/cleaned_data.txt" |
| 65 | + cat "$METRICS_FILEPATH" | |
| 66 | + grep -e "+Inf" -e "total" | |
| 67 | + grep -v "workqueue" | |
| 68 | + grep -v "go_" | |
| 69 | + grep -v "Identity" | |
| 70 | + grep -v "Capabili" | |
| 71 | + grep -v "TYPE" | |
| 72 | + grep -v "HELP" | |
| 73 | + grep -v "cloudprovider" | |
| 74 | + grep -v "promhttp" | |
| 75 | + grep -v "registered_metrics" >"$EXPORT_DIR/cleaned_data.txt" |
| 76 | +} |
| 77 | + |
| 78 | +(return 0 2>/dev/null) || ( |
| 79 | + echo "This script is not meant to be run directly, only sourced as a helper!" |
| 80 | + exit 1 |
| 81 | +) |
0 commit comments