a-stream

#!/bin/bash 

#default user
user=($USER)

#read static variables
inst_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $inst_root/a_env_conf
source $inst_root/allas-lib

# local variable
object_name=""              #object to retrieve
print_help=0
os_project_name="$OS_PROJECT_NAME"     
mode="swift"
tmp_dir="${tmp_root}/a_get_$$_tmp"
show_filelist=0
query=""
object_with_bucket=0
to_orig_dir=0
orig_dir=""
target_directory="."
asis_mode=0
exe_dir=$(pwd)
start_time=$(date +%s)
encrypt=""
secret_key=""

# read customer defaults
if [[ -e $HOME/.a_tools_conf ]]; then
   echo "Reading customer settings"  >&2
   source $HOME/.a_tools_conf
fi


#Process command line
while [[ $# -ge 1 ]]
do
  case "$1" in
             '--object' | '-o')
             # query file
                  object_name=($2)
                  shift
                  shift
                ;;
             '--project' | '-p' )
                  os_project_name=($2)
                  shift
                  shift
                ;;
             '--asis' )
                asis_mode=1
                  shift
                ;; 
             '--s3cmd' | '--s3allas' )
                  mode=("s3cmd")
                  shift
                ;;
             '--lumi' )
                mode=("lumi")
                shift
                ;;
             '--allas' )
                mode=("swift")
                shift
                ;;

             '--sk' )
                  secret_key="$2"
                  if [[ ! -e $secret_key ]]; then
                     echo "Error!"  >&2
                     echo "Secret key: $secret_key not found."  >&2
                     exit 1
                  fi
                  shift
                  shift
                ;;
             '-h' | '--help' )
                  print_help=(1)
                  shift
              ;;
             *)
                   if [[ $object_name != "" ]]; then
                      echo "unknown option: $1"  >&2
                      echo "Object name is: $object" >&2
                      exit 1
                   fi
                   object_name="$1"
                   shift                       # No more switches
                ;;
    esac
done



if [ $print_help -eq 1 ]; then
cat <<EOF
This tool is used to stream data, that has been uploaded to Allas service using the a-put command.
The basic syntax of the command is:

   a-stream object_name

By default the object is retrieved and uncompressed 

Options:

-p, --project <project_ID>    Search matches form the buckets of the defined project instead of the currently configured project. 

--asis                        Download the object without unpacking tar files and uncompressing zst compressed data.

--sk <secret key>             Secret key to open crypt4gh encryption.

--s3cmd                       Use S3 protocol and s3cmd command for data retrieval instead of Swift protocol and rclone.

Related commands: a-put, a-find, a-info, a-delete

EOF

exit 0

fi 

if [[ $object_name == "" ]]
then
  echo "Please give the object to retrieve:"  >&2 
  read object_name 
fi

project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)

#Check if connection works
if [[ $mode == "swift" ]]
then
  #here we check that rclone is available
  if [[ $(which rclone 2> /dev/null | wc -l ) -ne 1 ]];then
      echo ""  >&2
      echo "rclone is not available!" >&2
      echo "Please install rclone."  >&2
      exit 1
  fi
  #connection test
  test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
  #test=$(swift stat 2> /dev/null | grep -c "Account:")
  if [[ $test -lt 1 ]]
  then 
    echo "No connection to Allas!"  >&2
    echo "Please try setting the connection again."  >&2
    echo "by running command:" >&2 
    echo "" >&2
    echo "   source $allas_conf_path" >&2
    exit 1
  fi 
fi

#Rclone through s3
if [[ $mode == "s3cmd" ]];then
   storage_server="s3allas"
fi

#Rclone through s3
if [[ $mode == "lumi" ]];then
   storage_server="lumi-o"
fi
#source /appl/opt/allas_conf
#input=("$1")

#check that object exists

check_os=$(rclone ls ${storage_server}:$object_name 2> /dev/null | wc -l)
if [ $check_os -ne 1 ]; then
  echo "Object name: $object_name not found in $storage_server." >&2
  echo "" >&2
  exit 1
else
  #echo object_with_bucket
  object_with_bucket=(1)
fi


# check all buckets if the bucket is not defined
if [[ $object_with_bucket -eq 0 ]]
then
  #define standard buckets 
  buckets=("${user}-${project_label}-MISC ${user}-${project_label}-HOME ${project_label}-puhti-SCRATCH ${project_label}-puhti-PROJAPPL ${user}-${project_label}-pub ${user}-${project_label}-flip")
  num_buckets=(0)

  # go through the buckets
  for bn in $buckets
  do 
    bucket_found=$(rclone ls ${storage_server}:$bn/$object_name 2> /dev/null | wc -l)
    if [ $bucket_found -eq 1 ]; then
       echo "Bucket: $bn contains object:$object_name"  >&2
      (( num_buckets = num_buckets + 1 ))
      bucket=("$bn")
    fi
  done
  if [[ $num_buckets -eq 0 ]]; then
     echo "" >&2
     echo "Could not find object: $object_name " >&2
     exit 1
  fi
  if [[ $num_buckets -gt 1 ]]; then
     echo "" >&2
     echo "Object $object_name was found in several buckets!" >&2
     echo "Please include bucket name in the object name" >&2
     exit 1
  fi
  
  #add bucket to the object name
  object_name=("$bucket/$object_name")
else
  #check if object exists
  bucket_found=$(rclone ls ${storage_server}:/$object_name 2> /dev/null | wc -l)
  if [ $bucket_found -ne 1 ]; then
     a-find 

     echo "" >&2
     echo "Object $object_name was not found" >&2
     echo "" >&2
     echo "You can list all the available objects with command:" >&2
     echo "   a-find" >&2
  fi
fi 
   


#assign name for output file if not defined 
is_compressed=0
#if encrypted, remove .gpg temporarily from the object name to make zst and tar detection work
if [[ ${object_name:(-4):4} == ".gpg" ]]; then
  encrypt="gpg"
  echo "GPG encrypted object" >&2
  if [[ ${object_name:(-9):9} == ".gpg.c4gh" ]]; then
    is_compressed=1
  fi
fi
if [[ ${object_name:(-5):5} == ".c4gh" ]]; then
  encrypt="crypt4gh"
  echo "Crypt4gh encrypted object"  >&2
  if [[ $secret_key == "" && $asis_mode -eq 0 ]]; then
        echo "The secret key for decompression is not defined"  >&2 
        echo "Define the key file with option: --sk " >&2
        exit
  fi
  if [[ ${object_name:(-9):9} == ".zst.c4gh" ]]; then
    is_compressed=1
  fi

fi


if [[ ${object_name:(-4):4} == ".zst" ]]; then
  is_compressed=1
  #check if zstdmt is available
   if [[ $(which zstdmt 2> /dev/null | wc -l ) -ne 1 ]];then
      echo ""  >&2     
      echo "The data object you are trying to retrieve is compressed" >&2
      echo "with program: zstdmt that is not found in this server!" >&2
      echo "" >&2
      echo "Please install and add zstdmt to your command path." >&2
      echo "" >&2
      exit 1
   fi 
fi


is_tar_file=(0)

if [[ ${object_name:(-8):8} == ".tar.zst" ]]; then
  is_tar_file=(1)
  is_compressed=(1)
fi 

if [[ ${object_name:(-4):4} == ".tar" ]]; then
  is_tar_file=(1)
  is_compressed=(0)
fi 

echo Object_name: $object_name >&2

if [[ $target_name == "" ]];then 
   if [[ $is_tar_file -eq 1 ]]; then
      if [[ $is_compressed -eq 1 ]]; then
          target_name=$(basename $object_name .tar.zst) 
      else
          target_name=$(basename $object_name .tar)
      fi
   else
      target_name=$(basename $object_name .zst)
   fi 
fi

if [[ -n "$OS_PASSWORD" ]]; then 
   if  [[ $silent -eq 0 ]] ; then     
          echo "Updating token" >&2
   fi
   source $allas_conf_path --user $user -k $OS_PROJECT_NAME -f 
fi


echo "Starting to stream data from $storage_server..." >&2


if [[ $is_tar_file -eq 1 ]]; then
  echo "Tar files can't be streamed with a-stream" >&2
  exit
fi
     
# decompress if needed
if [[ $is_compressed -eq 1 ]]; then 
   if [[ $encrypt == "gpg" ]]; then
      rclone cat "${storage_server}:$object_name" | gpg -d | zstdmt  -d --stdout 
   elif [[  $encrypt == "crypt4gh" ]]; then
      rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk "$secret_key" | zstdmt -d --stdout 
   else
       rclone cat "${storage_server}:$object_name" | zstdmt -d --stdout 
   fi
  
else 
# Just download    
  if [[ $encrypt == "gpg" ]]; then
       rclone cat "${storage_server}:$object_name" | gpg -d 
  elif [[  $encrypt == "crypt4gh" ]]; then
       rclone cat "${storage_server}:$object_name" | crypt4gh decrypt --sk $secret_key
  else
       rclone cat "${storage_server}:$object_name" 
  fi
fi 


exit