a-get

#!/bin/bash 

#default user
user=($USER)

#read static variables
inst_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $inst_root/a_env_conf
source $inst_root/allas-lib

# local variable
object_name=""              #object to retrieve
file_name=""                #file to retrieve
target_name=""              #where to store the results
new_target=0                #do we create a new directory
new_target_file=0
print_help=0
os_project_name="$OS_PROJECT_NAME"     
mode="swift"
#tmp_dir="${tmp_root}/a_get_$$_tmp"
show_filelist=0
query=""
object_with_bucket=0
to_orig_dir=0
orig_dir=""
target_directory="."
asis_mode=0
exe_dir=$(pwd)
start_time=$(date +%s)
encrypt=""
secret_key=""
sdc=0

# read customer defaults
if [[ -e $HOME/.a_tools_conf ]]; then
   echo "Reading customer settings"
   source $HOME/.a_tools_conf
fi


#Process command line
while [[ $# -ge 1 ]]
do
  case "$1" in
             '--object' | '-o')
             # query file
                  object_name="$2"
                  shift
                  shift
                ;;
             '--project' | '-p' )
                  os_project_name="$2"
                  shift
                  shift
                ;;
             '--target_file' | '-t' )
                  target_name=$(basename $2)
                  if [[ $(dirname $2) != "." ]]; then
                     target_dir=$(dirname $2)
                     new_target=1
                  fi
                  new_target_file=1
                  shift
                  shift
                ;;
             '--target_dir' | '-d' )
                  target_dir="$2"
                  new_target=1
                  shift
                  shift
                ;;
             '--file' |  '-f' )
                  file_name="$2"
                  shift
                  shift
                ;;
             '--original_location' |  '-l' )
                  to_orig_dir=1                  
                  shift
                ;;
             '--asis' )
                asis_mode=1
                  shift
                ;; 
              '--s3cmd' )
                  mode="s3cmd"
                  shift
                ;;
              '--lumi' | '-L' )
                  mode="lumi"
                  shift
                ;;
              '--allas' | '-A' )
                  mode="swift"
                  shift
                ;;

              '--sk' )
                  secret_key="$2"
                  if [[ ! -e $secret_key ]]; then
                     echo "Error!"
                     echo "Secret key: $secret_key not found."
                     exit 1
                  fi
                  shift
                  shift
                ;;
             '--tmpdir' | '-T' )
                  tmp_root="$2"
                  shift
                  shift
                  ;;
	     '--sdc' )
		 sdc=1
                 if [[ $(which sd-lock-util 2> /dev/null | wc -l ) -ne 1 ]];then
		       echo ""
		       echo "sd-lock-util is not available!"
		       echo "Please install sd-lock-util if you want to use SD Connect based encryption."
		       echo "   https://github.com/CSCfi/sd-lock-util "
		       exit 1
		 fi
		 shift
		;;
             '-h' | '--help' )
                  print_help=1
                  shift
              ;;
             *)
                   if [[ $object_name != "" ]]; then
                      echo "unknown option: $1"
                      echo "Object name is: $object"
                      exit 1
                   fi
                   object_name="$1"
                   shift                       # No more switches
                ;;
    esac
done



if [ $print_help -eq 1 ]; then
cat <<EOF
This tool is used to download data that has been uploaded to Allas service using the a-put command.
The basic syntax of the command is:

   a-get object_name

By default the object is retrieved and decompressed 

Options:

-p, --project <project_ID>    Search matches form the buckets of the defined project instead of the currently configured project. 


-f, --file <file_name>        Retrieve just a specific file or directory from the stored dataset. Note that you need to define
                              the full path of the file or directory within the stored object.

-d, --target_dir <dir_name>   If this option is defined, a new target directory is created and the data is retrieved there.

-t, --target_file <file_name> Define a file name for the object for the object to be downloaded.

-l, --original_location       Retrieve the data to the original location in the directory structure.

--asis                        Download the object without unpacking tar files and decompressing zst compressed data.

--sk <secret key>             Secret key to open crypt4gh encryption.

-T, --tmpdir                  Define a directory that will be used to store 
                              temporary files of the download process.


--allas                       Get data from Allas with swift protocol in stead of currently set storage server. 
                              Normally this (Allas with swft) is the default and this option is not needed,
                              but if you have set e.g. Lumi-O as the default storage server, this option can be
                              used to get data from Allas without changing the default storage server.
                              
--s3cmd                       Use Allas with S3 protocol.

--lumi                        Get data from Lumi-O with S3 protocol in stead of the default storage server. 
                              If Lumi-O is defined to be the default storage server and this option is not needed.
                           

Related commands: a-put, a-find, a-info, a-delete

EOF

exit 0

fi 

if [[ $object_name == "" ]]
then
  echo "Please give the object to retrieve:"
  read object_name 
fi


#source $HOME/.allas_default
project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)

#Check if connection works
if [[ $mode == "swift" ]]
  then
  storage_server="allas"
  storage_name="Allas"
  #here we check that rclone is available
  if [[ $(which rclone 2> /dev/null | wc -l ) -ne 1 ]];then
      echo ""
      echo "rclone is not available!"
      echo "Please install rclone."
      exit 1
  fi
  #connection test
  test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
  #test=$(swift stat 2> /dev/null | grep -c "Account:")
  if [[ $test -lt 1 ]]
  then 
    echo "No connection to $storage_server!"
    echo "Please try setting up the connection again"
    echo "by running command:"
    echo ""
    echo "   source $allas_conf_path"
    exit 1
  fi 
fi


#chek SD Connect settings if that is used
if [[ $sdc -eq 1 ]]; then
    sdc_check=$(sd-lock-util pubkey | grep -c "BEGIN CRYPT4GH PUBLIC KEY")
    if [[ $sdc_check -ne 1 ]]; then
	echo "Connection to SD Connect service is not working"
	echo "Please open or refresh the connection"
	echo "by running command:"
        echo ""
        echo "   source $allas_conf_path -sdc"
        exit 1	
    fi
fi

#Rclone through s3
if [[ $mode == "s3cmd" ]];then
   storage_server="s3allas"
   storage_name="Allas"
fi

#lumi
if [[ $mode == "lumi" ]];then
   storage_server="lumi-o"
   storage_name="Lumi-o"
fi




#source /appl/opt/allas_conf
#input=("$1")

#check that object exists

check_os=$(rclone ls ${storage_server}:$object_name 2> /dev/null | wc -l)
if [ $check_os -ne 1 ]; then
  echo "Object name: $object_name not found in $storage_server."
  echo ""
  objects=($(a-find | grep "$object_name"))
  if [ ${#objects[@]} -eq 1 ]; then
    echo "Did you mean this object:"
    echo "  $objects"
  fi
  if  [ ${#objects[@]} -gt 1 ]; then   
    echo "Did you mean some of these objects:"
    for on in ${objects[@]}
    do 
        echo "  $on"
    done
  fi
  if [ ${#objects[@]} -eq 0 ]; then
    echo "Try running command:"
    echo "   a-find -a $object_name"
  fi
  exit 1
else
  #echo object_with_bucket
  object_with_bucket=(1)
fi


# check all buckets if the bucket is not defined
if [[ $object_with_bucket -eq 0 ]]
then
  #define standard buckets 
  buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip")
  num_buckets=(0)

  # go through the buckets
  for bn in $buckets
  do 
    bucket_found=$(rclone ls ${storage_server}:$bn/$object_name 2> /dev/null | wc -l)
    if [ $bucket_found -eq 1 ]; then
       echo "Bucket: $bn contains object:$object_name"
      (( num_buckets = num_buckets + 1 ))
      bucket=("$bn")
    fi
  done
  if [[ $num_buckets -eq 0 ]]; then
     echo ""
     echo "Could not find object: $object_name "
     exit 1
  fi
  if [[ $num_buckets -gt 1 ]]; then
     echo ""
     echo "Object $object_name was found in several buckets!"
     echo "Please include bucket name in the object name"
     exit 1
  fi
  
  #add bucket to the object name
  object_name=("$bucket/$object_name")
else
  #check if object exists
  bucket_found=$(rclone ls ${storage_server}:/$object_name 2> /dev/null | wc -l)
  if [ $bucket_found -ne 1 ]; then
     a-find 

     echo ""
     echo "Object $object_name was not found"
     echo ""
     echo "You can list all the available objects with command:"
     echo "   a-find"
  fi
fi 
   


#assign name for output file if not defined 

#if encrypted, remove .gpg temporarily from the object name to make zst and tar detection work
if [[ ${object_name:(-4):4} == ".gpg" ]]; then
  encrypt="gpg"
  object_name=${object_name:0:-4}
  echo "GPG encrypted object"
fi

if [[ ${object_name:(-5):5} == ".c4gh" ]]; then
  encrypt="crypt4gh"
  object_name=${object_name:0:-5}
  echo "Crypt4gh encrypted object"
  if [[ $sdc -eq 0 && $asis_mode -eq 0 ]]; then
     if [[ $secret_key == "" ]]; then
         #echo "Secret key not defied for Crypt4gh encrypted file."
         #echo "Checking if this is a SD Connect file"
	 sdc=1
         #check sd-locku-util is istalled
	 if [[ $(which sd-lock-util 2> /dev/null | wc -l ) -ne 1 ]];then
	    echo ""
	    echo "sd-lock-util is not available!"
	    echo "Please define the secretkey with option --sk if you are using normal Crypt4gh encryption ,"
	    echo "or install sd-lock-util if you are using SD Connect based encryption."
	    echo "   https://github.com/CSCfi/sd-lock-util "
	    exit 1
	 fi
	 # check that sd-lock connecton is wroking
             sdc_check=$(sd-lock-util pubkey | grep -c "BEGIN CRYPT4GH PUBLIC KEY")
         if [[ $sdc_check -ne 1 ]]; then
	    echo "Decryption is not possible."
	    echo "If you are using using normal Crypt4gh encryption, define the secretkey with option:"
            echo "   --sk your-secret-key"
	    echo ""
	    echo "If you are using SD Connect based encryption, refresh the sd-lock-util connection by running command:"
            echo ""
            echo "   source $allas_conf_path -sdc"
            exit 1	
         fi		 
     fi
  fi
fi

is_compressed=0

if [[ ${object_name:(-4):4} == ".zst" ]]; then
  is_compressed=1
  #check if zstdmt is available
   if [[ $(which zstdmt 2> /dev/null | wc -l ) -ne 1 ]];then
      echo ""      
      echo "The data object you are trying to retrieve is compressed"
      echo "with program: zstdmt that is not found in this server!"
      echo ""
      echo "Please install and add zstdmt to your command path."
      echo ""
      exit 1
   fi 
fi


is_tar_file=(0)

if [[ ${object_name:(-8):8} == ".tar.zst" ]]; then
  is_tar_file=(1)
  is_compressed=(1)
fi 

if [[ ${object_name:(-4):4} == ".tar" ]]; then
  is_tar_file=(1)
  is_compressed=(0)
fi 

if [[ $target_name == "" ]];then 
   if [[ $is_tar_file -eq 1 ]]; then
      if [[ $is_compressed -eq 1 ]]; then
          target_name=$(basename $object_name .tar.zst) 
      else
          target_name=$(basename $object_name .tar)
      fi
   else
      target_name=$(basename $object_name .zst)
   fi 
fi

if  [[ $encrypt == "gpg" ]]; then
  #return .gpg to the name
  object_name=${object_name}.gpg
fi

if  [[ $encrypt == "crypt4gh" || $sdc -eq 1 ]]; then
  #return .c4gh to the name
  object_name=${object_name}.c4gh
fi

#If selected, extract to the same place were data was from.
if [[ to_orig_dir -eq 1 ]]; then
   echo "orig dir:"
   rclone cat  ${storage_server}:${object_name}_ameta | awk '{ if ( $1 == "original_location:" ) print $2}'
   orig_dir=$(dirname $(rclone cat  ${storage_server}:${object_name}_ameta | awk '{ if ( $1 == "original_location:" ) print $2}'))
   mkdir -p $orig_dir
   cd $orig_dir
fi

# Reset target name, tar and compression information if requested with --asis
if [[ $asis_mode -eq 1 ]]; then
   is_tar_file=0
   is_compressed=0
   encrypt=""
   target_name=$(basename $object_name)
fi


#Test if the target file already exists
#echo "Target: ${target_dir}/${target_name}"
if [[ ${target_dir} == "" ]]; then
  if [[ -e ${target_name} ]]; then
    echo "File name: ${target_name} already exists!"
    echo "Please remove $target_name or define a different target directory name with option: --target_dir " 
    exit 1
  fi
else
   if [[ -e ${target_dir}/${target_name} ]]; then
      echo "File name:  ${target_dir}/${target_name} already exists!"
      echo "Please remove ${target_dir}/$target_name or define a different target directory name with option: --target_dir " 
   exit 1
   fi
fi
  

if [[ -n "$OS_PASSWORD" && $sdc -eq 0 ]]; then 
   if  [[ $silent -eq 0 ]] ; then     
          echo "Updating token"
   fi
   source $allas_conf_path --user $user -k $OS_PROJECT_NAME -f 
fi



echo "Starting to copy data from $storage_server..."


if [[ $is_tar_file -eq 1 ]]; then
     #Processing of a tar/directory
     
     if [[ $new_target -eq 1 ]];then
       if [[ -e $target_dir ]]; then
          check_dir=$(file -b $target_dir)
       else
          mkdir -p $target_dir
       fi
       if [[ $(file -b $target_dir | awk '{print $NF}') != "directory" ]]; then
            echo "Unable to create directory: $target_dir"
            exit 1
       fi

       cd $target_dir
     fi
     #rclone cat "${storage_server}:/$object_name" | zstdmt -d | tar -x -o $file_name
     echo "Downloading compressed object from $storage_name"
     
     #if [[ $mode == "swift" ]]; then
        rclone copyto -P "${storage_server}:$object_name" a-get_download_$$_tmp.zst
        exitcode=$?
     #else
     #  s3cmd get "s3://${object_name}" a-get_download_$$_tmp.zst 
     #fi

     if [[ $exitcode -ne 0 ]]; then
        echo "Download failed!"
        rm -f  a-get_download_$$_tmp.zst
        exit 1
     fi
     if [[ $encrypt == "gpg" ]]; then
        echo "Based on the object name, the object has been encrypted with gpg."
        gpg -d a-get_download_$$_tmp.zst > a-get_download_$$_tmp.zst.tmp
        exitcode=$?          
        rm -f a-get_download_$$_tmp.zst
        mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
     fi
     if [[ $encrypt == "crypt4gh" && $sdc -eq 0 ]]; then
        crypt4gh decrypt --sk "$secret_key" < a-get_download_$$_tmp.zst > a-get_download_$$_tmp.zst.tmp
        exitcode=$?
        if [[ $exitcode -ne 0 ]]; then
            # if decryption failed continue to check if this is SD
	    rm -f download_$$_tmp.zst.tmp
            sdc=1
	    
	else
	    #renaming successfully decrypted file.
            rm -f a-get_download_$$_tmp.zst
            mv a-get_download_$$_tmp.zst.tmp a-get_download_$$_tmp.zst
	fi
     fi
     if [[ $sdc -eq 1 ]]; then
	 locked_file=$(basename "$object_name")
	 locked_path=$(dirname  "$object_name")
         mv  a-get_download_$$_tmp.zst "$locked_file"
	 echo "Decrypting $locked-file"
	 sd-lock-util unlock --container "${locked_path}" --no-content-download --no-preserve-original --path "${locked_file}"
	 exitcode=$?
         unlocked_file="${locked_file:0:(-5)}"
	 if [[ -e "$unlocked_file" ]]; then
            mv "$unlocked_file"  a-get_download_$$_tmp.zst
            rm -f "$locked_file"
	 else
             echo "Decryption of $unlocked_file failed."
	     echo "This is probably now SD Connect encrypted file"
	     exitcode=1
	 fi
     fi
	 
     if [[ $exitcode -ne 0 ]]; then
        echo "Decryption failed!"
        rm -f  a-get_download_$$_tmp.zst
        exit 1
     fi


     echo "Uncompressing downloaded object"
     if [[ $is_compressed -eq 1 ]]; then
         cat  a-get_download_$$_tmp.zst | zstdmt -d | tar -xv -o $file_name
         exitcode=$?
     else
         cat  a-get_download_$$_tmp.zst | tar -xv -o $file_name
         exitcode=$?
     fi

     if [[ $exitcode -ne 0 ]]; then
        echo "Decompression failed!"
        rm -f  a-get_download_$$_tmp.zst
        exit 1
     fi
     rm -f  a-get_download_$$_tmp.zst
     if [[ $new_target -eq 1 ]];then
       cd -   
     fi
else
    #processing of a file
    if [[ $new_target -eq 1 ]];then
       if [[ -e $target_dir ]]; then
          check_dir=$(file -b $target_dir)
       else
          mkdir -p $target_dir
       fi
       if [[ $(file -b $target_dir | awk '{print $NF}' ) != "directory" ]]; then
            echo "Unable to create directory: $target_dir"
            exit 1
       fi
       cd $target_dir
    fi
    # decompress if needed
    if [[ $is_compressed -eq 1 ]]; then 
       #if [[ $mode == "swift" ]]; then 
           if [[ $encrypt == "gpg" ]]; then
             rclone cat "${storage_server}:$object_name" | gpg -d | zstdmt -d >  $target_name
             exitcode=$?
           elif [[  $encrypt == "crypt4gh" || $sdc -eq 0  ]]; then
             rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk "$secret_key" | zstdmt -d >  $target_name
             exitcode=$?
	   elif [[ $sdc -eq 1 ]]; then
               locked_file=$(basename "$object_name")
	       locked_path=$(dirname "$object_name")
	       echo "Copying encrypted data"
	       rclone copyto -P "${storage_server}:$object_name" "./${locked_file}"
	       echo "Decrypting $locked-file"
	       #echo " sd-lock-util unlock --container "${locked_path}" --no-content-download --no-preserve-original --path "${locked_file}" "
	       sd-lock-util unlock --container "${locked_path}" --no-content-download --no-preserve-original --path "${locked_file}"
	       exitcode=$?
               unlocked_file="${locked_file:0:(-5)}"
	       if [[ -e "$unlocked_file" ]]; then
	         if [[  "$unlocked_file" != "$target_name" ]]; then
	             cat "$unlocked_file" | zstdmt -d >  "$target_name"
                     rm -r "$unlocked_file"
	         fi
	       else
		  echo "Decrypting $unlocked_file failed"
		  exitcode=1 
	       fi	 
	       rm -f "$locked_file"	       
	   else
             rclone cat "${storage_server}:$object_name" | zstdmt -d >  $target_name
             exitcode=$?
           fi
    else 
       if [[ $target_dir == "" ]]; then 
            target_dir="."
       fi
       # Just download
       #if [[ $mode == "swift" ]]; then 
	   if [[ $encrypt == "gpg" ]]; then
             rclone cat "${storage_server}:$object_name" | gpg -d >  $target_name
           elif [[  $encrypt == "crypt4gh" && $sdc -eq 0 ]]; then
             #echo "rclone cat ${storage_server}:$object_name  crypt4gh decrypt --sk $secret_key"
             rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk "$secret_key" >  $target_name
             exitcode=$?	     
           elif [[ $sdc -eq 1 ]]; then
	       locked_file=$(basename "$object_name")
	       locked_path=$(dirname "$object_name")
	       echo "Copying encrypted data"
	       rclone copyto -P "${storage_server}:$object_name" "./${locked_file}"
	       echo "Decrypting $locked-file"
	       echo " sd-lock-util unlock --container "${locked_path}" --no-content-download --no-preserve-original --path "${locked_file}" "
	       sd-lock-util unlock --container "${locked_path}" --no-content-download --no-preserve-original --path "${locked_file}"
	       exitcode=$?
               unlocked_file="${locked_file:0:(-5)}"
               if [[ -e "$unlocked_file" ]]; then
         	   if [[  "$unlocked_file" != "$target_name" ]]; then
	               mv "$unlocked_file" "$target_name"
	           fi
	       else
		   echo "SD connect based decryption failed."
		   exitcode=1
	       fi
	       rm -f "$locked_file"
           else
             if [[ ${target_name:0:1} != "/" ]]; then
                #this is needed to cope with file names containing :
                rclone -P copyto "${storage_server}:$object_name"  "./$target_name"
                exitcode=$?
                fix_target_name=1
             else
                rclone -P copyto "${storage_server}:$object_name"  $target_name
                exitcode=$?
             fi
           fi

       #else
       #    s3cmd get "s3://${object_name}"  $target_dir/$target_name
       #    if [[ $encrypted == "gpg" ]]; then
       #       echo "Based on the object name, that object has been encrypted with gpg."
       #       gpg -d $target_dir/$target_name >  $target_dir/${target_name}.tmp       
       #       rm $target_dir/$target_name
       #       mv $target_dir/${target_name}.tmp $target_dir/$target_name
       #    fi 
       #    if [[ $encrypted == "crypt4gh" ]]; then
       #       echo "Based on the object name, that object has been encrypted with crypt4gh."
       #       crypt4gh decrypt --sk "$secret_key" < $target_dir/$target_name >  $target_dir/${target_name}.tmp       
       #       rm $target_dir/$target_name
       #       mv $target_dir/${target_name}.tmp $target_dir/$target_name
       #    fi 
       #fi
    fi

    if [[ $exitcode -ne 0 ]]; then
        echo "Download failed!"
        rm -f  $target_name
        exit 1
     fi

    if [[ $new_target -eq 1 ]];then
       cd -   
    fi
fi 
echo ""
echo "Object:"
echo "  $object_name "
echo "copied and uncompressed from $storage_server into:"
if [[ $target_dir == "" ]]; then
   echo "  ${orig_dir}${target_name}"
else
   echo "  ${orig_dir}${target_dir}/${target_name}"
fi
cd $exe_dir

end_time=$(date +%s)
(( kesto = end_time - start_time ))

#Execute log creation
message="$0 $(date) $kesto s"
printf '{"version": "1.1", "host": "%s", "short_message": "utility log", "full_message": "%s", "level": 6, "_user": "%d"}' $(hostname) "$message" $(id -u) >> $allas_log


#If log is a file and not a service then check permissions
if [[ $(ls $allas_log 2> /dev/null | wc -l) -eq 1 ]]; then
  if [[ $(ls -l $allas_log | awk '{print $3}') == $user ]]; then
     chmod a+rwx  $allas_log
  fi
fi


exit