Skip to content

Commit f7632ba

Browse files
mcapuccinimcapuccini
committed
Add formatting checks, and travis file refactory
1 parent 1565ffb commit f7632ba

File tree

7 files changed

+309
-274
lines changed

7 files changed

+309
-274
lines changed

‎.travis.yml‎

Lines changed: 41 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -1,34 +1,54 @@
1-
language: C
2-
3-
notifications:
4-
email: false
1+
---
2+
language: python # pip needed
53

64
sudo: required
75

6+
services:
7+
- docker
8+
9+
addons:
10+
apt:
11+
sources:
12+
- debian-sid
13+
packages:
14+
- shellcheck # code review tool
15+
816
env:
9-
matrix:
10-
- HOST_CLOUD=gce_clean_images.sh
11-
- HOST_CLOUD=aws_clean_images.sh
12-
- HOST_CLOUD=os_clean_images.sh
13-
- HOST_CLOUD=az_clean_images.sh
17+
global:
18+
- YAMLLINT_VERSION=1.8.1
19+
matrix:
20+
- CRON_SCRIPT=gce_clean_images.sh
21+
- CRON_SCRIPT=aws_clean_images.sh
22+
- CRON_SCRIPT=os_clean_images.sh
23+
- CRON_SCRIPT=az_clean_images.sh
1424

1525
before_install:
16-
# Due to the following Bug: https://github.com/travis-ci/travis-ci/issues/7940
17-
# To be removed in future revisions once bug is solved
18-
- sudo rm -f /etc/boto.cfg
26+
# Due to the following Bug: https://github.com/travis-ci/travis-ci/issues/7940
27+
# To be removed in future revisions once bug is solved
28+
- sudo rm -f /etc/boto.cfg
29+
30+
# Install code review tools
31+
- sudo pip install --upgrade pip
32+
- >
33+
sudo pip install
34+
ansible-lint=="${ANSIBLE_LINT_VERSION}"
35+
yamllint=="${YAMLLINT_VERSION}"
1936
20-
install: true
37+
# Check
38+
# check Shell
39+
- shellcheck *.sh
40+
- docker run -v "$(pwd)":/sh -w /sh jamesmstone/shfmt -i 2 -w *.sh
41+
- git diff --exit-code
42+
# check YAML
43+
- yamllint -c .yamllint.yml -s .travis.yml
2144

22-
before_script:
23-
# Installing common tools (pip and jq) and check bash scripts
24-
- sudo apt-get update
25-
- sudo apt-get install shellcheck jq python-pip -y
26-
- pip -V
27-
- pip install --upgrade pip
28-
- shellcheck $(find . -type f -name "*.sh")
45+
install:
46+
# Installing jq
47+
- sudo apt-get update
48+
- sudo apt-get install jq -y
2949

3050
script:
31-
- ./$HOST_CLOUD
51+
- ./$CRON_SCRIPT
3252

3353
notifications:
3454
email: false

‎.yamllint.yml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
---
2+
extends: default
3+
rules:
4+
comments:
5+
min-spaces-from-content: 1
6+
line-length:
7+
max: 150
8+
braces:
9+
min-spaces-inside: 0
10+
max-spaces-inside: 0
11+
brackets:
12+
min-spaces-inside: 0
13+
max-spaces-inside: 0
14+
indentation:
15+
indent-sequences: consistent

‎aws_clean_images.sh‎

Lines changed: 79 additions & 79 deletions
Original file line numberDiff line numberDiff line change
@@ -24,59 +24,59 @@ echo -e "Amazon Web Services - Looking for old KubeNow's AMIs:\n "
2424

2525
# Now we start the process of deregistering the old Kubenow AMI across all the other regions
2626
for reg in ${aws_regions[*]}; do
27-
28-
# We update the default region so to correctly perform checks in each region via awscli
29-
export AWS_DEFAULT_REGION="$reg"
30-
printf "Current region is: %s\n" "$AWS_DEFAULT_REGION"
31-
32-
# Extracting both KubeNow images that are flagged as "test" or "current"
33-
# Using tee and wc -l (which often returns 0) because of set -e. In case aws and grep return 1
34-
aws ec2 describe-images --filters "Name=name,Values=kubenow-*-*" "Name=owner-id,Values=105135433346" | tee /tmp/aws_out_images.json
35-
tot_no_amis=$(grep -i "imageid" < /tmp/aws_out_images.json | wc -l)
36-
counter_del_img=0
37-
counter_del_snap=0
38-
39-
if [ "$tot_no_amis" -gt "0" ]; then
40-
# Finding Image IDs of AMIs older than 1 day which needed to be deregistered
41-
index=0
42-
while [ "$index" -lt "$tot_no_amis" ]; do
43-
44-
img_date=$(jq ".Images[$index] | .CreationDate" /tmp/aws_out_images.json | sed -e 's/^"//' -e 's/"$//' -e 's/T.*//')
45-
46-
if [[ ! "$img_date" > "$del_date" ]]; then
47-
# Extracting AMI's "Name" and "ImageId"
48-
name=$(jq ".Images[$index] | .Name" /tmp/aws_out_images.json | sed -e 's/^"//' -e 's/"$//')
49-
ami_id_to_deregister=$(jq ".Images[$index] | .ImageId" /tmp/aws_out_images.json | sed -e 's/^"//' -e 's/"$//')
50-
echo -e "Following old AMI dated $img_date is found \nName: $name \nAMI ID:$ami_id_to_deregister\n"
51-
52-
# Find if there are any snapshots attached to the Image need to be deregister
53-
aws ec2 describe-images --image-ids "$ami_id_to_deregister" | grep snap | awk ' { print $2 }' | sed -e 's/^"//' -e 's/,$//' -e 's/"$//' > /tmp/old_snaps.txt
54-
55-
# Deregistering the AMI
56-
echo -e "Starting the deregister of KubewNow AMI: $ami_id_to_deregister...\n"
57-
aws ec2 deregister-image --image-id "$ami_id_to_deregister"
58-
counter_del_img=$((counter_del_img+1))
59-
60-
# Deleting snapshots attached to AMI
61-
while read -r line; do
62-
echo -e "Deleting the associated snapshots: $line \n"
63-
aws ec2 delete-snapshot --snapshot-id "$line"
64-
counter_del_snap=$((counter_del_snap+1))
65-
done < /tmp/old_snaps.txt
66-
fi
67-
68-
index=$((index+1))
69-
done
70-
71-
if [ "$counter_del_img" == "0" ]; then
72-
echo -e "No old images dated $del_date were found \n"
73-
fi
74-
75-
else
76-
echo -e "No KubeNow AMIs flagged as test or current found"
77-
fi
78-
79-
echo -e "\nNo of deleted of AMI: $counter_del_img\nNo of deleted snapshots: $counter_del_snap\n"
27+
28+
# We update the default region so to correctly perform checks in each region via awscli
29+
export AWS_DEFAULT_REGION="$reg"
30+
printf "Current region is: %s\n" "$AWS_DEFAULT_REGION"
31+
32+
# Extracting both KubeNow images that are flagged as "test" or "current"
33+
# Using tee and wc -l (which often returns 0) because of set -e. In case aws and grep return 1
34+
aws ec2 describe-images --filters "Name=name,Values=kubenow-*-*" "Name=owner-id,Values=105135433346" | tee /tmp/aws_out_images.json
35+
tot_no_amis=$(grep -i "imageid" </tmp/aws_out_images.json | wc -l)
36+
counter_del_img=0
37+
counter_del_snap=0
38+
39+
if [ "$tot_no_amis" -gt "0" ]; then
40+
# Finding Image IDs of AMIs older than 1 day which needed to be deregistered
41+
index=0
42+
while [ "$index" -lt "$tot_no_amis" ]; do
43+
44+
img_date=$(jq ".Images[$index] | .CreationDate" /tmp/aws_out_images.json | sed -e 's/^"//' -e 's/"$//' -e 's/T.*//')
45+
46+
if [[ ! "$img_date" > "$del_date" ]]; then
47+
# Extracting AMI's "Name" and "ImageId"
48+
name=$(jq ".Images[$index] | .Name" /tmp/aws_out_images.json | sed -e 's/^"//' -e 's/"$//')
49+
ami_id_to_deregister=$(jq ".Images[$index] | .ImageId" /tmp/aws_out_images.json | sed -e 's/^"//' -e 's/"$//')
50+
echo -e "Following old AMI dated $img_date is found \nName: $name \nAMI ID:$ami_id_to_deregister\n"
51+
52+
# Find if there are any snapshots attached to the Image need to be deregister
53+
aws ec2 describe-images --image-ids "$ami_id_to_deregister" | grep snap | awk ' { print $2 }' | sed -e 's/^"//' -e 's/,$//' -e 's/"$//' >/tmp/old_snaps.txt
54+
55+
# Deregistering the AMI
56+
echo -e "Starting the deregister of KubewNow AMI: $ami_id_to_deregister...\n"
57+
aws ec2 deregister-image --image-id "$ami_id_to_deregister"
58+
counter_del_img=$((counter_del_img + 1))
59+
60+
# Deleting snapshots attached to AMI
61+
while read -r line; do
62+
echo -e "Deleting the associated snapshots: $line \n"
63+
aws ec2 delete-snapshot --snapshot-id "$line"
64+
counter_del_snap=$((counter_del_snap + 1))
65+
done </tmp/old_snaps.txt
66+
fi
67+
68+
index=$((index + 1))
69+
done
70+
71+
if [ "$counter_del_img" == "0" ]; then
72+
echo -e "No old images dated $del_date were found \n"
73+
fi
74+
75+
else
76+
echo -e "No KubeNow AMIs flagged as test or current found"
77+
fi
78+
79+
echo -e "\nNo of deleted of AMI: $counter_del_img\nNo of deleted snapshots: $counter_del_snap\n"
8080
done
8181

8282
# Now taking care of the S3 buckets for KubeNow images
@@ -85,38 +85,38 @@ s3_buckets=("us-east-1" "eu-central-1")
8585
# Extracting List of objects from each bucket
8686
for buck in ${s3_buckets[*]}; do
8787

88-
echo -e "AWS S3 $buck - Looking for old KubeNow bucket objects:\n"
89-
# Technicality here about the set -e at the beginning at grep's exit code -1. Thus using tee which no matter the outcomes, will return 0
90-
aws s3 ls s3://kubenow-"$buck" --region "$buck" --human-readable | grep -E 'kubenow-v([0-9]*)([ab0-9]*)-([0-9]*)-([a-z0-9]*)-([test]*)([current]*).qcow2([.md5]*)' | tee /tmp/aws_s3_objs.txt
88+
echo -e "AWS S3 $buck - Looking for old KubeNow bucket objects:\n"
89+
# Technicality here about the set -e at the beginning at grep's exit code -1. Thus using tee which no matter the outcomes, will return 0
90+
aws s3 ls s3://kubenow-"$buck" --region "$buck" --human-readable | grep -E 'kubenow-v([0-9]*)([ab0-9]*)-([0-9]*)-([a-z0-9]*)-([test]*)([current]*).qcow2([.md5]*)' | tee /tmp/aws_s3_objs.txt
9191

92-
no_obj_to_check=$(wc -l < /tmp/aws_s3_objs.txt)
93-
echo -e "No of bucket object to be checked: $no_obj_to_check\n"
92+
no_obj_to_check=$(wc -l </tmp/aws_s3_objs.txt)
93+
echo -e "No of bucket object to be checked: $no_obj_to_check\n"
9494

95-
counter_del_s3_obj=0
95+
counter_del_s3_obj=0
9696

97-
if [ "$no_obj_to_check" -gt "0" ]; then
97+
if [ "$no_obj_to_check" -gt "0" ]; then
9898

99-
while read -r line; do
100-
obj_date=$(echo "$line" | awk '{print $1}')
101-
obj_name=$(echo "$line" | awk '{print $5}')
102-
103-
if [[ ! "$obj_date" > "$del_date" ]]; then
104-
echo -e "Following old KubeNow bucket object dated: $obj_date is found\nName: $obj_name\n"
105-
echo -e "Starting the delete bucket object: $obj_name...\n"
106-
aws s3 rm "s3://kubenow-$buck/$obj_name" --region "$buck"
107-
counter_del_s3_obj=$((counter_del_s3_obj+1))
108-
fi
109-
done < /tmp/aws_s3_objs.txt
110-
111-
if [ "$counter_del_s3_obj" == "0" ]; then
112-
echo -e "No old bucket objects dated $del_date were found \n"
113-
fi
114-
115-
else
116-
echo -e "\nNo KubeNow bucket objects flagged as test or current found"
99+
while read -r line; do
100+
obj_date=$(echo "$line" | awk '{print $1}')
101+
obj_name=$(echo "$line" | awk '{print $5}')
102+
103+
if [[ ! "$obj_date" > "$del_date" ]]; then
104+
echo -e "Following old KubeNow bucket object dated: $obj_date is found\nName: $obj_name\n"
105+
echo -e "Starting the delete bucket object: $obj_name...\n"
106+
aws s3 rm "s3://kubenow-$buck/$obj_name" --region "$buck"
107+
counter_del_s3_obj=$((counter_del_s3_obj + 1))
108+
fi
109+
done </tmp/aws_s3_objs.txt
110+
111+
if [ "$counter_del_s3_obj" == "0" ]; then
112+
echo -e "No old bucket objects dated $del_date were found \n"
117113
fi
118114

119-
echo -e "\nNo of deleted bucket object: $counter_del_s3_obj\n"
115+
else
116+
echo -e "\nNo KubeNow bucket objects flagged as test or current found"
117+
fi
118+
119+
echo -e "\nNo of deleted bucket object: $counter_del_s3_obj\n"
120120
done
121121

122122
echo -e "Done.\n"

‎aws_del_old_snaps.sh‎

Lines changed: 62 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -9,66 +9,66 @@ echo -e "Amazon Web Services - Looking for old unpaired snaposhots:\n "
99

1010
# Now we start the process of deregistering the old Kubenow AMI across all the other regions
1111
for reg in ${aws_regions[*]}; do
12-
13-
# We update the default region so to correctly perform checks in each region via awscli
14-
AWS_DEFAULT_REGION="$reg"
15-
echo -e "Current region is: $AWS_DEFAULT_REGION\n"
16-
17-
# Extracting both KubeNow images that are flagged as "test" or "current"
18-
aws ec2 describe-snapshots --owner-ids 105135433346 --query 'Snapshots[*].{ID:SnapshotId,Description:Description}' > /tmp/aws_snaps.json
19-
sed -i '1s/^/{"Snapshots":/' /tmp/aws_snaps.json
20-
sed -i "$ a }" /tmp/aws_snaps.json
21-
tot_no_snaps=$(grep -c -i ID < /tmp/aws_snaps.json)
22-
counter_del_snap=0
23-
24-
if [ "$tot_no_snaps" -gt "0" ]; then
25-
# Finding if the AMI which the current snapshot has been created for still exist. If not, we can delete snapshot
26-
echo -e "Total of current snapshot is: $tot_no_snaps\n"
27-
28-
index=0
29-
while [ "$index" -lt "$tot_no_snaps" ]; do
30-
31-
# Extracting AMI's ID from snapshot's description field
32-
ami_id_to_check=$(jq ".Snapshots[$index] | .Description" /tmp/aws_snaps.json | grep -E -o -m1 "ami-([a-z0-9]*)")
33-
snap_id=$(jq ".Snapshots[$index] | .ID" /tmp/aws_snaps.json | sed -e 's/^"//' -e 's/"$//')
34-
35-
# When copying an AMI from one region to another, both the source and the destination AMI ids are listed. We need to keep and check only first id
36-
no_of_rel_amis=$(echo "$ami_id_to_check" | wc -w)
37-
38-
if [ "$no_of_rel_amis" == "0" ] && [ -z "$ami_id_to_check" ]; then
39-
# This should never happen. However if something goes wrong, then we avoid to call APIs which will fail otherwise
40-
echo -e "Oops. Something went wrong at this point.\nNo of AMI ids to be checked seems zero. This should not be the case\n"
41-
exit 1
42-
elif [ "$no_of_rel_amis" -gt "1" ]; then
43-
ami_id_to_check=$(echo "$ami_id_to_check" | awk NR==1'{print $1}')
44-
fi
45-
46-
echo -e "Snapshot id: $snap_id"
47-
echo -e "Related AMI Id to be checked: $ami_id_to_check"
48-
49-
# Checking whether or not AMI still exists
50-
aws ec2 describe-images --image-ids "$ami_id_to_check" > /tmp/output_AMI_check 2> /tmp/out_err_AMI_check
51-
aws_exit_code=$?
52-
aws_string_err=$(grep -o "does not exist" < /tmp/out_err_AMI_check)
53-
54-
if [ "$aws_exit_code" == "0" ]; then
55-
echo -e "AMI id: $ami_id_to_check still exists. Snapshot id: $snap_id will not be deleted.\n"
56-
elif [ "$aws_exit_code" == "255" ] && [ -n "$aws_string_err" ]; then
57-
echo -e "AMI id: $ami_id_to_check does not exist anymore. Snapshot id: $snap_id will be deleted.\n"
58-
echo -e "Deleting the snapshots id: $snap_id \n"
59-
aws ec2 delete-snapshot --snapshot-id "$snap_id"
60-
counter_del_snap=$((counter_del_snap+1))
61-
fi
62-
63-
index=$((index+1))
64-
done
65-
66-
if [ "$counter_del_snap" == "0" ]; then
67-
echo -e "No old unpaired snapshots were found \n"
68-
fi
69-
else
70-
echo -e "There are no available snapshots in the current region: $AWS_DEFAULT_REGION"
71-
fi
72-
73-
echo -e "\nNo of deleted of snapshots: $counter_del_snap\nDone.\n"
12+
13+
# We update the default region so to correctly perform checks in each region via awscli
14+
AWS_DEFAULT_REGION="$reg"
15+
echo -e "Current region is: $AWS_DEFAULT_REGION\n"
16+
17+
# Extracting both KubeNow images that are flagged as "test" or "current"
18+
aws ec2 describe-snapshots --owner-ids 105135433346 --query 'Snapshots[*].{ID:SnapshotId,Description:Description}' >/tmp/aws_snaps.json
19+
sed -i '1s/^/{"Snapshots":/' /tmp/aws_snaps.json
20+
sed -i "$ a }" /tmp/aws_snaps.json
21+
tot_no_snaps=$(grep -c -i ID </tmp/aws_snaps.json)
22+
counter_del_snap=0
23+
24+
if [ "$tot_no_snaps" -gt "0" ]; then
25+
# Finding if the AMI which the current snapshot has been created for still exist. If not, we can delete snapshot
26+
echo -e "Total of current snapshot is: $tot_no_snaps\n"
27+
28+
index=0
29+
while [ "$index" -lt "$tot_no_snaps" ]; do
30+
31+
# Extracting AMI's ID from snapshot's description field
32+
ami_id_to_check=$(jq ".Snapshots[$index] | .Description" /tmp/aws_snaps.json | grep -E -o -m1 "ami-([a-z0-9]*)")
33+
snap_id=$(jq ".Snapshots[$index] | .ID" /tmp/aws_snaps.json | sed -e 's/^"//' -e 's/"$//')
34+
35+
# When copying an AMI from one region to another, both the source and the destination AMI ids are listed. We need to keep and check only first id
36+
no_of_rel_amis=$(echo "$ami_id_to_check" | wc -w)
37+
38+
if [ "$no_of_rel_amis" == "0" ] && [ -z "$ami_id_to_check" ]; then
39+
# This should never happen. However if something goes wrong, then we avoid to call APIs which will fail otherwise
40+
echo -e "Oops. Something went wrong at this point.\nNo of AMI ids to be checked seems zero. This should not be the case\n"
41+
exit 1
42+
elif [ "$no_of_rel_amis" -gt "1" ]; then
43+
ami_id_to_check=$(echo "$ami_id_to_check" | awk NR==1'{print $1}')
44+
fi
45+
46+
echo -e "Snapshot id: $snap_id"
47+
echo -e "Related AMI Id to be checked: $ami_id_to_check"
48+
49+
# Checking whether or not AMI still exists
50+
aws ec2 describe-images --image-ids "$ami_id_to_check" >/tmp/output_AMI_check 2>/tmp/out_err_AMI_check
51+
aws_exit_code=$?
52+
aws_string_err=$(grep -o "does not exist" </tmp/out_err_AMI_check)
53+
54+
if [ "$aws_exit_code" == "0" ]; then
55+
echo -e "AMI id: $ami_id_to_check still exists. Snapshot id: $snap_id will not be deleted.\n"
56+
elif [ "$aws_exit_code" == "255" ] && [ -n "$aws_string_err" ]; then
57+
echo -e "AMI id: $ami_id_to_check does not exist anymore. Snapshot id: $snap_id will be deleted.\n"
58+
echo -e "Deleting the snapshots id: $snap_id \n"
59+
aws ec2 delete-snapshot --snapshot-id "$snap_id"
60+
counter_del_snap=$((counter_del_snap + 1))
61+
fi
62+
63+
index=$((index + 1))
64+
done
65+
66+
if [ "$counter_del_snap" == "0" ]; then
67+
echo -e "No old unpaired snapshots were found \n"
68+
fi
69+
else
70+
echo -e "There are no available snapshots in the current region: $AWS_DEFAULT_REGION"
71+
fi
72+
73+
echo -e "\nNo of deleted of snapshots: $counter_del_snap\nDone.\n"
7474
done

0 commit comments

Comments
 (0)