a-find

#!/bin/bash

#rclone is needed
if [[ $(which rclone 2> /dev/null | wc -l ) -ne 1 ]];then
      echo ""
      echo "rclone is not available!"
      echo "Please install rclone."
      exit 1
fi


#read static variables
inst_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $inst_root/a_env_conf
source $inst_root/allas-lib

#local variables
user="$USER"
bucket_name="not_defined"
silent=0
print_help=0
os_project_name="$OS_PROJECT_NAME"
input_def=""
mode="swift"
#tmp_dir=("${tmp_root}/a_put_$$_tmp")
show_filelist=0
query=""
all_buckets=0


# read customer defaults
if [[ -e $HOME/.a_tools_conf ]]; then
   customized=1
   source $HOME/.a_tools_conf
else
   customized=0
fi

#Process command line
while [[ $# -ge 1 ]]
do
  case "$1" in
             '--user' | '-u' )
                  user="$2"
                  shift
                  shift
                ;;
             '--bucket' | '-b' )
             # query file
                  bucket_name=($2)
                  shift
                  shift
                ;;
             '--project' | '-p')
                  os_project_name=($2)
                  shift
                  shift
                ;;
             '-silent' | '-s')
                  silent=1
                  shift
                ;;
            '--lumi')
                  mode="lumi"
                  shift
                ;;
            '--files' | '-f' )
                  show_filelist=1
                  shift
                ;;
            '--all_bukets' | '-a' | '--all_buckets' )
                  all_buckets=1
                  shift
                ;;
             '-h' | '--help' )
                  print_help=1
                  shift
                ;;
             *)
                   if [[ $query != "" ]]; then
                      echo "unknown option: $1"
                      echo "Query term is: $query"
                      exit 1
                   fi
                   query="$1"
                   shift                       # No more switches
                ;;
    esac
done

# note about customization
if [[ $silent -eq 0 ]]; then
   if [[ $customized -eq 1 ]]; then
     echo "Customer settings read from $HOME/.a_tools_conf"
   fi
fi



#echo "buk $bucket_name"

#exit

if [ $print_help -eq 1 ]; then
cat <<EOF
This tool is used to find:
 1. Objects, whose name matches to the query
 2. a-put generated objects that contain matching file names.

The basic syntax of the command is:

   a-find query_term


The query term is compared to the object names as well as the names and original paths of the files that
have been uploaded to Allas with a-put. The matching objects are reported (but not downloaded).

The query term is processed as a regular expression where some characters, for example dot (.), have a special meaning.
The same regular expression syntax is used with e.g. grep, awk and sed commands.
The most commonly occurring special characters are listed below:

    the dot (.) is used to define any single character.
    ^ means the beginning of a line
    $ means the end of a line
    [ ] matches any of the characters inside the brackets. For example [abc] would match a,b or c.
    [^ ] matches any character, except the characters inside the brackets.
    For example [^abc] would select all rows that contain also other characters
    than just a,b and c.
    * matches zero or more of the preceding character or expression
    \{n,m\} matches n to m occurrences of the preceding character or expression

Options:

-f, --files                  Lists the names of the matching files inside the objects in addition to the object name.

-p, --project <project_ID>   Search matches form the buckets of the defined project instead of the currently configured project.

-b, --bucket <bucket_name>   By default all the standard buckets, used by a-put, are searched. Option --bucket allows you to specify a
                        single bucket that will be used for the search.

-a, --all                    By default all the standard buckets, used by a-put, are searched. Option --all defines
                             that all the buckets of the project will be included in the search.

-s, --silent                 Output just the object names and number of hits. If -file option is included,
                             print object name and matching file name on one row.


Related commands: a-put, a-get, a-delete, a-info
EOF

exit

fi

#Assign project to be used if not defined
if [ "$os_project_name" == "" ]
then
  if [ -f $HOME/.allas_default ]
  then
     source $HOME/.allas_default
  else
     echo "Default project is not defined"
     source $allas_conf_path -user $user
     echo "os_project_name=$OS_PROJECT_NAME" > $HOME/.allas_default
     echo "Default allas project is stored to  \$HOME/.allas_default"
     echo ""
     source $HOME/.allas_default
  fi
fi


project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)

#Check if connection works
if [[ $mode == "swift" ]]
then
  storage_server="allas"
  test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
  #test=$(swift stat 2> /dev/null | grep -c "Account:")
  if [[ $test -lt 1 ]]
  then
    echo "No connection to Allas!"
    echo "Please try setting up the connection again"
    echo "by running the command:"
    echo ""
    echo "   source $allas_conf_path"
    exit 1
  fi
fi


if [[ $mode == "s3cmf" ]]
then
  storage_server="s3allas"
fi

if [[ $mode == "lumi" ]]
then
  storage_server="lumi-o"
fi

if [[ $all_buckets -eq 1 ]]; then
 #bucket_name=("$(swift list| tr  '\n' ' ')")
 bucket_name=("$(rclone lsd  ${storage_server}: | awk '{print $NF}'| tr  '\n' ' ')")
fi

#define standard buckets
std_buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip")

#check all buckets if a specific bucket is not defined
if [[ "$bucket_name" == "not_defined" ]]
then
  buckets=$std_buckets
  #echo $buckets
else
  buckets=$bucket_name
fi



#list all objects if query is not defined
if [[ $query == "" ]]
then
  echo ""
  echo "No query term given!"
  echo "Listing all objects but not files within them:"
  echo " "

  for bn in $buckets
  do
    for metafile in $(rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep "_ameta$")
    do
        os_dirname=$(dirname "$metafile" )
        if [ $os_dirname == "." ]; then
           os_dirname=""
        else
           os_dirname="/$os_dirname"
        fi
        os_filename=$(basename "$metafile" _ameta)
        os_object=("${bn}${os_dirname}/${os_filename}")
        echo $os_object
    done
  done
  exit
fi




#source /appl/opt/allas_conf
#input=("$1")

total_hits=(0)
total_objects=(0)
object_hits=(0)
total_object_hits=(0)
for bn in $buckets
do
  if [ $silent -eq 0 ]; then
    echo "----------------------------------------------"
    echo "Checking bucket: $bn"
  fi
  rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print "x " $0}' | tr -s ' '|  cut -d " " -f 3- | grep -v "_ameta$" | grep "$query" > ./$$_find_tmp
  object_hits=$(cat ./$$_find_tmp | wc -l)
  if [[ $object_hits -gt 0 ]];then
    if [ $silent -eq 0 ]; then
      echo ""
      echo "Matching object names:"
    fi
    awk '{ print "'$bn/'" $0}' ./$$_find_tmp ; rm -f ./$$_find_tmp
    (( total_object_hits = total_object_hits + object_hits ))
    if [ $silent -eq 0 ]; then
      echo "----------------------------------------------"
      echo "a-put generated objects that include matching file name:";echo ""
    fi

  fi

  for metafile in $(rclone ls ${storage_server}:$bn 2> /dev/null | awk '{print $2}' | grep "_ameta$")
  do
     num_hits=$(rclone cat ${storage_server}:${bn}/$metafile | awk '{print $9}' | grep -c "$query")
     if [[ $num_hits -gt 0 ]]; then
        os_dirname=$(dirname "$metafile" )
        if [ $os_dirname == "." ]; then
           os_dirname=""
        else
           os_dirname="/$os_dirname"
        fi
        os_filename=$(basename "$metafile" _ameta)
        os_object=("${bn}${os_dirname}/${os_filename}")
        if [ $show_filelist -eq 1 ]; then
           if [ $silent -eq 0 ]; then
              echo "Matching files in object $os_object :"
              rclone cat ${storage_server}:${bn}/$metafile | awk '{print $9}' | grep "$query"
              echo ""
           else
              #silent output
              rclone cat ${storage_server}:${bn}/$metafile | awk '{print $9}' | grep "$query" | awk '{print "'$os_object' "$0}'
           fi
        else
           if [ $silent -eq 0 ]; then
             echo ""
             #echo " Object:  ${bn}/$(dirname "$metafile")/$(basename $metafile _ameta)"
             echo " Object: $os_object"
             echo " includes $num_hits file names that that match query: $query"
             echo ""
           else
             echo $os_object $num_hits
           fi
        fi
        (( total_objects = total_objects + 1 ))
        (( total_hits = total_hits + num_hits ))
     fi
  done

done

if [ $silent -eq 0 ]; then
  echo "------------------------------------------------ "
  echo "Query: $query"
  echo "Number of object names matching: $total_object_hits"
  echo ""
  echo "Total of $total_hits matching file names were found"
  echo "in $total_objects objects uploaded with a-put"
  echo "-------------------------------------------------"
fi
exit