allas-health-check

#!/bin/bash


#read static variables
inst_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
#inst_root="/appl/opt/allas-cli-utils"
#cat  $inst_root/a_env_conf
source $inst_root/a_env_conf
source $inst_root/allas-lib

allas_conf_path=("${inst_root}/allas_conf")

export OS_USERNAME=$(whoami)
export OS_AUTH_URL=https://pouta.csc.fi:5001/v3
export OS_USER_DOMAIN_NAME="Default"
export OS_REGION_NAME="regionOne"
export OS_INTERFACE=public
export OS_IDENTITY_API_VERSION=3
project="none"
bucket=""
#store swift authentication if it is done
if [ -n "$OS_AUTH_TOKEN" ]
then
   ORIG_OS_AUTH_TOKEN=("$OS_AUTH_TOKEN")
fi 

if [ -n "$OS_STORAGE_URL" ]
then
   ORIG_OS_STORAGE_UR=("$OS_STORAGE_URL")
fi 

buk=(1)

usage="

allas-health-check checks segment related metadata for all segmented objects in all projects. 
With the option -p, you can focus the checking to just one project.

OPTIONS 

  -p, --project    Do testing only for a specific project.
  -b, --bucket     Do testing only for a specific bucket. Project must be explicitly given with -p .
"


while [[ $# -ge 1 ]]
do
  case "$1" in
             '-p' | '--project')
             project="$2"
             shift
             shift
             ;;
             '-b' | '--bucket')
             bucket="$2"
             shift
             shift
             ;;
             *)
             echo "$usage"
             exit 1
             shift                       # No more switches
             ;;
    esac
done



if [ -z "$OS_PASSWORD" ]; then 
   echo "Please enter CSC password for account ${OS_USERNAME}: "
   read -sr OS_PASSWORD_INPUT
   export OS_PASSWORD=$OS_PASSWORD_INPUT
fi

if [[ $project == "none" ]]; then
   echo "Collecting project list. Please wait."
   openstack project list --my-projects -c Name -f value
   all_projects=$(openstack project list --my-projects -c Name -f value)
else
   all_projects="$project"
fi

fail_tot=0



for pro in $all_projects
do
  #Open connection
  echo "-----------------------------------------------"
  projnum=$( echo $pro | sed -e s/"project_"/""/g | sed -e s/"Project_"/""/g )
  saldo -p $projnum 2> /dev/null | grep "^Project"
  # source $allas_conf_path -f  --keeppasswd $pro
  source $allas_conf_path -f --silent --keeppasswd $pro > /dev/null
  echo "-------------------------------------------" 
  echo "Project: $pro"
  echo "-------------------------------------------" >>  /tmp/allas-health-tmp-$$ 
  echo "Project: $pro" >> /tmp/allas-health-tmp-$$
  num_buckets=$(swift stat | grep 'Containers:' | awk '{ print $2}')
 
  if [[ $num_buckets -gt 999 ]]; then
     echo "Maxmium number of buckets reached: $num_buckets"
  else
      echo "Number of buckets: $num_buckets"
  fi
   
  #Run the test  
  fail_pro=0
  echo $bucket
  if [[ $bucket == "" ]]; then
     bucketlist=$(rclone lsd ${storage_server}: | grep _segments$  | awk '{print $NF}' | sed 's/.\{9\}$//')
     #echo "rclone lsd ${storage_server}: | grep _segments$  | awk '{print $NF}' | sed 's/.\{9\}$//'"
  else
     bucketlist=$bucket
  fi

  #Go through all the buckets that have segments bucket
  for bucket in $bucketlist
  do 
       echo "Checking bucket: $bucket"
       num_ok=0
       num_fail=0

      #check that bucket and segments bucket has same owner
      auth=$(swift stat ${bucket} | grep "Account:"  | awk '{print $NF}')
      auth_seg=$(swift stat ${bucket}_segments | grep "Account:"  | awk '{print $NF}')
      if [[ $auth == "" ]]; then
         echo "Main bucket that would match ${bucket}_segments was not found!"
      else 
         if [[ $auth != $auth_seg ]]; then
            echo "$bucket and  ${bucket}_segments are owned by different projects"
         fi
      fi        

 

       #List segmented objects
       #echo "rclone ls "${storage_server}:${bucket}_segments" | cut -d " " -f 2| sed 's/.\{9\}$//' | uniq"
      ## for sobject in $(rclone ls "${storage_server}:${bucket}_segments" | grep -v "/slo/" | sed  s/"^ "/""/ | cut -d " " -f 2 | sed 's/.\{9\}$//' | uniq )
      ## do
      ##   fobject=$( echo "$sobject" | rev | cut -d "/" -f 3-  | rev )
      for fobject in $(rclone ls "${storage_server}:${bucket}_segments" | grep -v "/slo/" | sed  s/"^ "/""/ | cut -d " " -f 2 | sed 's/.\{9\}$//' | rev | cut -d "/" -f 3-  | rev | sort |uniq )
      do
      ## do


         # this check does not work for segment names with /slo/
         slo_object=$(echo "$fobject" | grep -c '/slo/')
         if [[ $slo_object -eq 0 ]]; then
           #echo Sobject: $sobject
           #echo Fobject: $fobject
           check_if_exists=$(rclone ls ${storage_server}:${bucket}/${fobject}| wc -l)
           if [[ $check_if_exists -eq 0 ]]; then
               echo "Object $fobject has segment objects, but no front object"         
               echo "    Object $bucket/$fobject has segment objects, but no front object."  >> /tmp/allas-health-tmp-$$ 
               (( num_fail = num_fail + 1 )) 
               (( fail_pro = fail_pro + 1 ))
           else
              #echo "check_segment_sizes ${storage_server} ${bucket} ${fobject}"
              seg_stat=$(check_segments ${storage_server} ${bucket} ${fobject})
              if [[ $seg_stat -lt 0 ]]; then
                  (( num_fail = num_fail + 1 )) 
                  (( fail_pro = fail_pro + 1 ))
                  echo  "Object $fobject has some segment objects missing."           
                  echo  "    Object $bucket/$fobject has some segment objects missing."  >> /tmp/allas-health-tmp-$$
              fi
              if [[ $seg_stat -gt 0 ]]; then
                  (( num_fail = num_fail + 1 )) 
                  (( fail_pro = fail_pro + 1 ))
                  mani_obj=$(swift stat ${bucket} ${fobject} | grep Manifest | awk '{print $2}')
                  meta_size=$(swift stat ${bucket} ${fobject} | grep "Content Length:" | awk '{print $3}')
                  segs_size=$(rclone ls ${storage_server}:$mani_obj | awk '{a = a + $1}END{print a}')
                  if [[ $meta_size -eq  $segs_size ]];then
                     echo  "Object $fobject is OK but it has extra segment objects that could be removed."           
                     echo  "    Object $fobject is OK but it has extra segment objects that could be removed."  >> /tmp/allas-health-tmp-$$
                     (( num_ok = num_ok + 1 ))
                  else
                     (( num_fail = num_fail + 1 )) 
                     (( fail_pro = fail_pro + 1 ))
                     echo  "Object $fobject has some segment objects missing!"           
                     echo  "     Object $bucket/$fobject has some segment objects missing."  >> /tmp/allas-health-tmp-$$
                  fi
              fi
              if [[ $seg_stat -eq 0 ]]; then
              ## bucket_check=$( swift stat ${bucket} ${fobject} | grep Manifest | grep -c "$sobject" ) 
              ##if [[ $bucket_check -ne 1 ]]; then      
              ##  echo "    Problem with object: ${bucket}/$fobject"
              ##   echo "    Problem with object: ${bucket}/$fobject" >>  /tmp/allas-health-tmp-$$
              ##   echo "    Segments are either not found or there are several versions available." >>  /tmp/allas-health-tmp-$$
              ##  (( num_fail = num_fail + 1 )) 
              ##  (( fail_pro = fail_pro + 1 ))
              ##else
              # echo "    $fobject OK"
                 (( num_ok = num_ok + 1 ))
              fi
           fi
         fi     
       done
     if [[ $num_fail -eq 0 ]]; then
        echo "All objects OK"
     else
        echo "$num_fail failed checks"
        (( fail_tot = fail_tot + num_fail ))
     fi
  done
  if [[ $fail_pro -eq 0 ]]; then
    echo "OK" >> /tmp/allas-health-tmp-$$
  fi
  bucket=""
done

if [[ fail_tot -gt 0 ]]; then
   echo "   "
   echo "----------------------------------------------------------------"
   echo "   "
   echo "----------------------------------------------------------------"
   echo "Summary of Allas health check"
   cat /tmp/allas-health-tmp-$$
   rm -f  /tmp/allas-health-tmp-$$
fi


#store swift authentication if it is done
if [ -n "$ORIG_OS_AUTH_TOKEN" ]
then
   export OS_AUTH_TOKEN=("$ORIG_OS_AUTH_TOKEN")
fi 

if [ -n "$ORIG_OS_STORAGE_URL" ]
then
   export OS_STORAGE_URL=("$ORIG_OS_STORAGE_URL")
fi 

#Execute log creation
message="$0 $(date) broken-objects: $fail_tot "
printf '{"version": "1.1", "host": "%s", "short_message": "utility log", "full_message": "%s", "level": 6, "_user": "%d"}' $(hostname) "$message" $(id -u) >> $allas_log


#If log is a file and not a service then check permissions
if [[ $(ls $allas_log 2> /dev/null | wc -l) -eq 1 ]]; then
  if [[ $(ls -l $allas_log | awk '{print $3}') == $user ]]; then
     chmod a+rwx  $allas_log
  fi
fi