Skip to content

Commit f3c6ce5

Browse files
brikis98brikis98
authored
Merge pull request #2 from gruntwork-io/aws-fixes
Add tests and fixes for AWS helpers
2 parents 4dfbca0 + 792b14f commit f3c6ce5

13 files changed

+1865
-28
lines changed

‎.circleci/Dockerfile

+16-5
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,24 @@
11
FROM ubuntu:16.04
22
MAINTAINER Gruntwork <[email protected]>
33

4-
# Install Bats
4+
# Install basic dependencies
55
RUN DEBIAN_FRONTEND=noninteractive apt-get update && \
6-
apt-get install -y software-properties-common && \
6+
apt-get install -y vim python-pip jq sudo curl
7+
8+
# Install Bats
9+
RUN apt-get install -y software-properties-common && \
710
add-apt-repository ppa:duggan/bats && \
811
DEBIAN_FRONTEND=noninteractive apt-get update && \
912
apt-get install -y bats
1013

11-
# Install other basic dependencies
12-
RUN apt-get install -y python-pip jq sudo && \
13-
pip install awscli --upgrade --user
14+
# Install AWS CLI
15+
RUN pip install awscli --upgrade --user
16+
17+
# Install moto: https://github.com/spulec/moto
18+
RUN pip install flask moto moto[server]
19+
20+
# Install tools we'll need to create a mock EC2 metadata server
21+
RUN apt-get install -y net-tools iptables
22+
23+
# Copy mock AWS CLI into the PATH
24+
COPY ./aws-local.sh /usr/local/bin/aws

‎.circleci/aws-local.sh

+16
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
# A wrapper script for the AWS CLI that redirects all calls to localhost:5000 so that they go to moto instead of the
3+
# real AWS servers. This script should be installed in the PATH so it gets called instead of the real AWS CLI, and this
4+
# script will, in turn, call the real AWS CLI.
5+
6+
set -e
7+
8+
# Set mock values so the AWS CLI doesn't complain
9+
export AWS_ACCESS_KEY_ID="mock-for-testing"
10+
export AWS_SECRET_ACCESS_KEY="mock-for-testing"
11+
export AWS_DEFAULT_REGION="us-east-1"
12+
13+
# We assume that the AWS CLI has been installed by pip into this path
14+
readonly REAL_AWS_CLI="$HOME/.local/bin/aws"
15+
16+
"$REAL_AWS_CLI" --endpoint-url=http://localhost:5000 "$@"

‎.circleci/config.yml

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,11 @@
11
version: 2
22
jobs:
33
build:
4-
docker:
5-
- image: gruntwork/bash-commons-circleci-tests
4+
# We need to run Docker Compose with privileged settings, which isn't supported by CircleCI's Docker executor, so
5+
# we have to use the machine executor instead.
6+
machine: true
67
steps:
78
- checkout
8-
- run: bats test
9+
- run: |
10+
cd test
11+
docker-compose up

‎.gitignore

+3
Original file line numberDiff line numberDiff line change
@@ -11,3 +11,6 @@
1111
build/
1212
*/build/
1313
out/
14+
15+
# Python
16+
*.pyc

‎modules/bash-commons/src/aws-wrapper.sh

+24-19
Original file line numberDiff line numberDiff line change
@@ -7,31 +7,34 @@ set -e
77

88
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/log.sh"
99
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/aws.sh"
10-
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/assertions.sh"
10+
source "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/assert.sh"
1111

12-
# Get the name of the ASG this EC2 Instance is in
12+
# Get the name of the ASG this EC2 Instance is in. This is done by looking up the instance's tags. This method will
13+
# wait up until the specified number of retries if the tags or instances are not yet available.
1314
function aws_wrapper_get_asg_name {
15+
local readonly max_retries="$1"
16+
local readonly sleep_between_retries="$2"
17+
1418
local instance_id
1519
instance_id=$(aws_get_instance_id)
1620

1721
local instance_region
1822
instance_region=$(aws_get_instance_region)
1923

20-
aws_wrapper_get_instance_tag "$instance_id" "$instance_region" "aws:autoscaling:groupName"
24+
aws_wrapper_get_instance_tag "$instance_id" "$instance_region" "aws:autoscaling:groupName" "$max_retries" "$sleep_between_retries"
2125
}
2226

2327
# Look up the tags for the specified Instance and extract the value for the specified tag key
2428
function aws_wrapper_get_instance_tag {
2529
local readonly instance_id="$1"
2630
local readonly instance_region="$2"
2731
local readonly tag_key="$3"
28-
29-
local readonly max_retries=60
30-
local readonly sleep_between_retries=5
32+
local readonly max_retries="${4:-60}"
33+
local readonly sleep_between_retries="${5:-5}"
3134

3235
for (( i=0; i<"$max_retries"; i++ )); do
3336
local tags
34-
tags=$(aws_wrapper_wait_for_instance_tags "$instance_id" "$instance_region")
37+
tags=$(aws_wrapper_wait_for_instance_tags "$instance_id" "$instance_region" "$max_retries" "$sleep_between_retries")
3538
assert_not_empty_or_null "$tags" "tags for Instance $instance_id in $instance_region"
3639

3740
local tag_value
@@ -57,12 +60,11 @@ function aws_wrapper_get_instance_tag {
5760
function aws_wrapper_wait_for_instance_tags {
5861
local readonly instance_id="$1"
5962
local readonly instance_region="$2"
63+
local readonly max_retries="${3:-60}"
64+
local readonly sleep_between_retries="${4:-5}"
6065

6166
log_info "Looking up tags for Instance $instance_id in $instance_region"
6267

63-
local readonly max_retries=60
64-
local readonly sleep_between_retries=5
65-
6668
for (( i=0; i<"$max_retries"; i++ )); do
6769
local tags
6870
tags=$(aws_get_instance_tags "$instance_id" "$instance_region")
@@ -88,9 +90,8 @@ function aws_wrapper_wait_for_instance_tags {
8890
function aws_wrapper_get_asg_size {
8991
local readonly asg_name="$1"
9092
local readonly aws_region="$2"
91-
92-
local readonly max_retries=60
93-
local readonly sleep_between_retries=5
93+
local readonly max_retries="${3:-60}"
94+
local readonly sleep_between_retries="${4:-5}"
9495

9596
for (( i=0; i<"$max_retries"; i++ )); do
9697
log_info "Looking up the size of the Auto Scaling Group $asg_name in $aws_region"
@@ -121,12 +122,12 @@ function aws_wrapper_get_asg_size {
121122
function aws_wrapper_wait_for_instances_in_asg {
122123
local readonly asg_name="$1"
123124
local readonly aws_region="$2"
125+
local readonly max_retries="${3:-60}"
126+
local readonly sleep_between_retries="${4:-5}"
124127

125128
local asg_size
126-
asg_size=$(aws_wrapper_get_asg_size "$asg_name" "$aws_region")
127-
128-
local readonly max_retries=60
129-
local readonly sleep_between_retries=5
129+
asg_size=$(aws_wrapper_get_asg_size "$asg_name" "$aws_region" "$max_retries" "$sleep_between_retries")
130+
assert_not_empty_or_null "$asg_size" "size of ASG $asg_name in $aws_region"
130131

131132
log_info "Looking up Instances in ASG $asg_name in $aws_region"
132133
for (( i=0; i<"$max_retries"; i++ )); do
@@ -157,9 +158,11 @@ function aws_wrapper_get_ips_in_asg {
157158
local readonly asg_name="$1"
158159
local readonly aws_region="$2"
159160
local readonly use_public_ips="$3"
161+
local readonly max_retries="${4:-60}"
162+
local readonly sleep_between_retries="${5:-5}"
160163

161164
local instances
162-
instances=$(aws_describe_instances_in_asg "$asg_name" "$aws_region")
165+
instances=$(aws_wrapper_wait_for_instances_in_asg "$asg_name" "$aws_region" "$max_retries" "$sleep_between_retries")
163166
assert_not_empty_or_null "$instances" "Get info about Instances in ASG $asg_name in $aws_region"
164167

165168
local readonly ip_param=$([[ "$use_public_ips" == "true" ]] && echo "PublicIpAddress" || echo "PrivateIpAddress")
@@ -172,9 +175,11 @@ function aws_wrapper_get_hostnames_in_asg {
172175
local readonly asg_name="$1"
173176
local readonly aws_region="$2"
174177
local readonly use_public_hostnames="$3"
178+
local readonly max_retries="${4:-60}"
179+
local readonly sleep_between_retries="${5:-5}"
175180

176181
local instances
177-
instances=$(aws_wrapper_wait_for_instances_in_asg "$asg_name" "$aws_region")
182+
instances=$(aws_wrapper_wait_for_instances_in_asg "$asg_name" "$aws_region" "$max_retries" "$sleep_between_retries")
178183
assert_not_empty_or_null "$instances" "Get info about Instances in ASG $asg_name in $aws_region"
179184

180185
local readonly hostname_param=$([[ "$use_public_hostnames" == "true" ]] && echo "PublicDnsName" || echo "PrivateDnsName")

‎test/aws-cli.bats

+124
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,124 @@
1+
#!/usr/bin/env bats
2+
3+
source "$BATS_TEST_DIRNAME/../modules/bash-commons/src/aws.sh"
4+
load "test-helper"
5+
load "aws-helper"
6+
7+
function setup {
8+
start_moto
9+
}
10+
11+
function teardown {
12+
stop_moto
13+
}
14+
15+
@test "aws_get_instance_tags empty" {
16+
run aws_get_instance_tags "fake-id" "us-east-1"
17+
assert_success
18+
19+
local readonly expected=$(cat <<END_HEREDOC
20+
{
21+
"Tags": []
22+
}
23+
END_HEREDOC
24+
)
25+
26+
assert_output_json "$expected"
27+
}
28+
29+
@test "aws_get_instance_tags non-empty" {
30+
local readonly tag_key="foo"
31+
local readonly tag_value="bar"
32+
33+
local instance_id
34+
instance_id=$(create_mock_instance_with_tags "$tag_key" "$tag_value")
35+
36+
run aws_get_instance_tags "$instance_id" "us-east-1"
37+
assert_success
38+
39+
local readonly expected=$(cat <<END_HEREDOC
40+
{
41+
"Tags": [
42+
{
43+
"ResourceType": "instance",
44+
"ResourceId": "$instance_id",
45+
"Value": "$tag_value",
46+
"Key": "$tag_key"
47+
}
48+
]
49+
}
50+
END_HEREDOC
51+
)
52+
53+
assert_output_json "$expected"
54+
}
55+
56+
@test "aws_describe_asg empty" {
57+
run aws_describe_asg "fake-asg-name" "us-east-1"
58+
assert_success
59+
60+
local readonly expected=$(cat <<END_HEREDOC
61+
{
62+
"AutoScalingGroups": []
63+
}
64+
END_HEREDOC
65+
)
66+
67+
assert_output_json "$expected"
68+
}
69+
70+
@test "aws_describe_asg non-empty" {
71+
local readonly asg_name="foo"
72+
local readonly min_size=1
73+
local readonly max_size=3
74+
local readonly region="us-east-1"
75+
local readonly azs="${region}a"
76+
77+
create_mock_asg "$asg_name" "$min_size" "$max_size" "$azs"
78+
79+
run aws_describe_asg "$asg_name" "$region"
80+
assert_success
81+
82+
local actual_asg_name
83+
actual_asg_name=$(echo "$output" | jq -r '.AutoScalingGroups[0].AutoScalingGroupName')
84+
assert_equal "$asg_name" "$actual_asg_name"
85+
86+
local actual_min_size
87+
actual_min_size=$(echo "$output" | jq -r '.AutoScalingGroups[0].MinSize')
88+
assert_equal "$min_size" "$actual_min_size"
89+
90+
local actual_max_size
91+
actual_max_size=$(echo "$output" | jq -r '.AutoScalingGroups[0].MaxSize')
92+
assert_equal "$max_size" "$actual_max_size"
93+
}
94+
95+
@test "aws_describe_instances_in_asg empty" {
96+
run aws_describe_instances_in_asg "fake-asg-name" "us-east-1"
97+
assert_success
98+
99+
local readonly expected=$(cat <<END_HEREDOC
100+
{
101+
"Reservations": []
102+
}
103+
END_HEREDOC
104+
)
105+
106+
assert_output_json "$expected"
107+
}
108+
109+
@test "aws_describe_instances_in_asg non-empty" {
110+
local readonly asg_name="foo"
111+
local readonly min_size=1
112+
local readonly max_size=3
113+
local readonly region="us-east-1"
114+
local readonly azs="${region}a"
115+
116+
create_mock_asg "$asg_name" "$min_size" "$max_size" "$azs"
117+
118+
run aws_describe_instances_in_asg "$asg_name" "$region"
119+
assert_success
120+
121+
local num_instances
122+
num_instances=$(echo "$output" | jq -r '.Reservations | length')
123+
assert_greater_than "$num_instances" 0
124+
}

‎test/aws-ec2-metadata.bats

+71
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
#!/usr/bin/env bats
2+
3+
source "$BATS_TEST_DIRNAME/../modules/bash-commons/src/aws.sh"
4+
load "test-helper"
5+
load "aws-helper"
6+
7+
readonly local_ipv4="11.22.33.44"
8+
readonly public_ipv4="55.66.77.88"
9+
readonly local_hostname="ip-10-251-50-12.ec2.internal"
10+
readonly public_hostname="ec2-203-0-113-25.compute-1.amazonaws.com"
11+
readonly instance_id="i-1234567890abcdef0"
12+
readonly mock_region="us-west-1"
13+
readonly availability_zone="${mock_region}b"
14+
15+
function setup {
16+
start_ec2_metadata_mock \
17+
"$local_ipv4" \
18+
"$public_ipv4" \
19+
"$local_hostname" \
20+
"$public_hostname" \
21+
"$instance_id" \
22+
"$mock_region" \
23+
"$availability_zone"
24+
}
25+
26+
function teardown {
27+
stop_ec2_metadata_mock
28+
}
29+
30+
@test "aws_get_instance_private_ip" {
31+
run aws_get_instance_private_ip
32+
assert_success
33+
assert_output "$local_ipv4"
34+
}
35+
36+
@test "aws_get_instance_public_ip" {
37+
run aws_get_instance_public_ip
38+
assert_success
39+
assert_output "$public_ipv4"
40+
}
41+
42+
@test "aws_get_instance_private_hostname" {
43+
run aws_get_instance_private_hostname
44+
assert_success
45+
assert_output "$local_hostname"
46+
}
47+
48+
@test "aws_get_instance_public_hostname" {
49+
run aws_get_instance_public_hostname
50+
assert_success
51+
assert_output "$public_hostname"
52+
}
53+
54+
@test "aws_get_instance_id" {
55+
run aws_get_instance_id
56+
assert_success
57+
assert_output "$instance_id"
58+
}
59+
60+
@test "aws_get_instance_region" {
61+
run aws_get_instance_region
62+
assert_success
63+
assert_output "$mock_region"
64+
}
65+
66+
@test "aws_get_ec2_instance_availability_zone" {
67+
run aws_get_ec2_instance_availability_zone
68+
assert_success
69+
assert_output "$availability_zone"
70+
}
71+

0 commit comments

Comments
 (0)