a-publish

#!/bin/bash 


#default user
user=($USER)

#read static variables
inst_root="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source $inst_root/a_env_conf
source $inst_root/allas-lib

if [[ $# -lt 1 ]]
then
  echo "Please give the name of a file to be uploaded to allas as an argument of this command."
  echo ""
  exit 1
fi

# local variables
include_ameta=0
index_mode="static" #static or dynamic
tmp_file="not_defined"
bucket_name="not_defined"
print_help=0
os_project_name="$OS_PROJECT_NAME"
input_def=""
mode="swift"
#tmp_dir="${tmp_root}/a_put_$$_tmp"
make_temp_dir

# read customer defaults
if [[ -e $HOME/.a_tools_conf ]]; then
   customized=1
   source $HOME/.a_tools_conf
else
   customized=0
fi



#Process command line
while [[ $# -ge 1 ]]
do
  case "$1" in
             '--bucket' | '-b' )
             # query file
                  bucket_name=($2)
                  shift
                  shift
                ;;
             '--project' | '-p' )
                  os_project_name=($2)
                  shift
                  shift
                ;;
             '--os_file' | '-o' )
                  partial_path=$(dirname $2)
                  tmp_file=$(basename $2)
                  shift
                  shift
                ;;
            '--s3cmd' )
                  mode=("s3cmd")
                  shift
                ;;
            '--lumi' )
                  mode=("lumi")
                  shift
                ;;
            '--allas' )
                  mode=("swift")
                  shift
                ;;
             '--index' | '-i' )
                  index_mode=$2
                  if [[ $index_mode == "static" || $index_mode == "dynamic" ]]; then
                     echo "creating $index_mode index"
                  else
                     echo "Unknown index mode: $index_mode"
                     echo "Should be: static or dynamic" 
                     exit 1
                  fi
                  shift
                  shift
                ;;
              '--input-list')
                  list_file=$2
                  if [[ -e $list_file ]];then
                    input_def=("$(cat $list_file)")                
                  else  
                    echo "Import file list $list_file not found"
                    exit 1  
                  fi 
                  shift
                  shift
                ;;
             '-h' | '--help' )
                  print_help=(1)
                  shift
                ;;
             *)
                   input_def=("$input_def $1")
                   shift                       # No more switches
                ;;
    esac
done

if [ $print_help -eq 1 ]; then
cat <<EOF

a-publish copies a file to Allas into a bucket that can be publicly accessed. Thus, anyone with the address (URL) of the 
uploaded data object can read and download the data with a web browser or tools like wget and curl. 
a-publish works mostly like a-put but there are some differences: 

1) a-publish can upload only files, not directories. 
2) files are not compressed but they uploaded as they are. 
3) the access control of the target bucket is set so that it is available in read-only mode to the internet.

The basic syntax of the command is:

  a-publish file_name

By default, the file is uploaded to a bucket username-projectNumber-pub. You can define other bucket names too using option -b
but you should note that this command will make all data in the bucket publicly accessible, 
including data that has been previously uploaded to the bucket.

The public URL to a data object is:

https://a3s.fi/username-projectNumber-pub/object_name

An object uploaded with a-publish can be removed from Allas with command a-delete.

a-publish options:

 -b, --bucket       Use the defined bucket instead of the default bucket name
 -o, --os_file      Define alternative name for the object that will be created  
 -i, --index        (static/dynamic).  By default a-publish creates a static index 
                    file that includes the objects that are in the target bucket when 
                    the command is executed.
                    With setting --index dynamic the command adds a JavaScript based 
                    index file to the bucket. With this option the index.html page 
                    lists the objects that are available in the bucket in the time when 
                    this page is accessed. This dynamic indexing tool can list
                    only up to 1000 files.
 --input-list       List of files to be uploaded.    
 

EOF

exit 

fi 

#Check if connection works
if [[ $mode == "swift" ]]
then
  test=$(rclone about ${storage_server}: 2> /dev/null | wc -l)
  #test=$(swift stat 2> /dev/null | grep -c "Account:")
  if [[ $test -lt 1 ]]
  then 
    echo "No connection to Allas!"
    echo "Please try setting up the connection again."
    exit 1
  fi 
fi


echo "Files to be uploaded: $input_def"

#set default bucket
project_label=$(echo ${os_project_name} |  sed -e s/"project_"/""/g)
if [[ "$bucket_name" == "not_defined" ]]
then
   bucket_name=("${user}-${project_label}-pub")
fi 

#In case of lumi the bucket must be cereated
if [[ $mode == "lumi" ]]; then
   storage_server="lumi-pub"
   echo "rclone mkdir ${storage_server}:${bucket_name}"
   rclone mkdir ${storage_server}:${bucket_name}
fi


echo "Bucket: $bucket_name"




for input in $input_def
do 
  echo "Processing: $input"
  if [ ! -e $input ] ; then
    echo "File: $input does not exist!"
    exit 1
  fi
  
  if [[ $(file -b $input | grep -c directory ) -eq 1 ]]
  then
    echo "This command can only be used to publish files, not directories."
    exit 1
  fi
 
  #check that file name does not end with _ameta
  if [[ ${input:(-5):5} == "_ameta" ]]; then
    echo "Found a file/directoryname which ends with _ameta"
    echo "  $input"
    echo ""
    echo "Please rename this file as it will mix up a the metadata management of a-put."
    exit 1
  fi

  file_path=$(abspath $input)

  echo "Checking total size of $input. Please wait."
  tot_size=$(du -s $input | cut -f1)
  #echo $tot_size

  #tmp file name. Depends on compression and is file is a directory
  if [  $tmp_file == "not_defined" ]
  then
     tmp_file=($(basename $input | tr " " "_" ))
  fi


  #Check if stored file already exists
  if [[ $(rclone ls ${storage_server}:${bucket_name}/$tmp_file 2> /dev/null | wc -c) -gt 2 ]]
  then
    echo ""
    echo "A file/directory with the same name has already been uploaded into"
    echo "bucket $bucket_name in $storage_server"
    echo ""
    rclone lsl ${storage_server}:${bucket_name}/$tmp_file  
    echo ""
    echo "Do you wish to overwrite the existing old object? [y/n]"
    read overwrite_old
    if [[ $overwrite_old == "y" ]] || [[ $overwrite_old == "yes" ]]; then
      echo "Overwriting ${bucket_name}/$tmp_file"
    else
      exit 1
    fi
  fi

  #collect and count metadata
  if [[ include_ameta -eq 1 ]]; then
    echo "user: $user" >> ${tmp_dir}/${tmp_file}_ameta
    echo "host: $(hostname)" >> ${tmp_dir}/${tmp_file}_ameta
    echo "original_location: $file_path" >>  ${tmp_dir}/${tmp_file}_ameta
    echo ""
    ls -l $input >>  ${tmp_dir}/${tmp_file}_ameta
  fi  

  if [[ $tot_size -gt $max_size ]]
  then 
    echo "This file or directory is too big for this tool"
    echo "Total size: ${tot_size}K"
    echo "Please use swift or rclone command to upload the data to $storage_server"
    rm -f ${tmp_dir}/${tmp_file}_ameta
    clean_temp_dir
    exit 1
  fi 

  ln -s ${file_path} ${tmp_dir}/$tmp_file

  #upload
  #if [[ $mode == "swift" ]]
  #then
  # For less than 5GB files rclone is used for uploading
    
  echo "Uploading data to allas."
  # echo "rclone copy --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}"
  rclone copy -L --progress ${tmp_dir}/$tmp_file ${storage_server}:${bucket_name}/${partial_path}
  exitcode=$?
  if [[ $exitcode -ne 0 ]]; then
     echo ""
     echo "File upload for $infile failed"
     if [[ $partial_path == "" ]]; then
        rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
     else
        rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
     fi
     rm -f ${tmp_dir}/$tmp_file 
     if [[ $include_ameta -eq 1  ]]; then
       rm -f ${tmp_dir}/${tmp_file}_ameta
     fi
     clean_temp_dir
     exit 1
  fi

  # rclone md5sums can be calculated only for files that are smaller than 5GB
  if [[ $tot_size -lt 5000000 ]]
  then
     echo "Confirming upload..."
     #checksums for local and allas files 
     sum1=$(checksum_tmpdir_tmpfile)
       
     if [[ $partial_path == "" ]]; then
        sum2=($(rclone md5sum  ${storage_server}:${bucket_name}/$tmp_file))
     else  
        sum2=($(rclone md5sum  ${storage_server}:${bucket_name}/${partial_path}/$tmp_file))
     fi
     #check if checksums match 
     if [[ ${sum1[0]} !=  ${sum2[0]} ]]
     then 
       echo "Upload of $input was not successful!"
       echo "Cleaning the failed upload"
       if [[ $partial_path == "" ]]; then
          rclone deletefile ${storage_server}:${bucket_name}/$tmp_file
       else
          rclone deletefile ${storage_server}:${bucket_name}/${partial_path}/$tmp_file
       fi
       rm -f ${tmp_dir}/$tmp_file 
       rm -f ${tmp_dir}/${tmp_file}_ameta
       clean_temp_dir
       exit 1
     fi
     echo "$input OK"
  fi


  #echo "Removing temporary file"
  rm -f  ${tmp_dir}/$tmp_file

  if [[ $include_ameta -eq 1 ]]; then
    #update metadata
    echo ""
    echo "Adding metadata for uploaded $input"
    #echo "rclone copy ./${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}" 
    rclone copy  ${tmp_dir}/${tmp_file}_ameta ${storage_server}:${bucket_name}/${partial_path}
  
    rm -f ${tmp_dir}/${tmp_file}_ameta
  fi

  echo "$input uploaded to ${bucket_name}"

  if [[ $mode == "swift" ]]; then
     https_root="https://a3s.fi"
  fi
  if [[ $mode == "lumi" ]]; then
     https_root="https://${os_project_name}.lumidata.eu"
  fi

  if [[ $partial_path == "" ]]; then 
     echo "Public link: ${https_root}/${bucket_name}/$tmp_file"
  else 
     echo "Public link: ${https_root}/${bucket_name}/${partial_path}/$tmp_file"
  fi
  tmp_file=("not_defined")

done

#Publish and rebuild index file
#Publish and rebuild index file
if [[ $index_mode == "static" ]];then
  echo '<html>' >  ${tmp_dir}/index.html
  echo '<table>' >>  ${tmp_dir}/index.html
  echo "<h2>Contents of bucket $bucket_name<h2>" >>  ${tmp_dir}/index.html
  rclone lsl ${storage_server}:$bucket_name | grep -v "_ameta" | awk '{print "<tr><td>"$1"</td><td>"$2"</td><td>"$3"</td><td><b><a href=\"'${https_root}'/'$bucket_name'/"$4"\">"$4"</a></b></td><td><a href=\"'${https_root}'/'$bucket_name'/"$4"_ameta\" target=\"_self\" >info</a></td></tr>"}' >>  ${tmp_dir}/index.html
  echo '</table>'  >>  ${tmp_dir}/index.html
  echo '</html>' >>  ${tmp_dir}/index.html
else
  wget https://raw.githubusercontent.com/CSCfi/allas-cli-utils/master/dynamic_index.js
  mv dynamic_index.js ${tmp_dir}/index.html
fi


rclone copy  ${tmp_dir}/index.html ${storage_server}:${bucket_name}/
rm -f   ${tmp_dir}/index.html

#set access control
if [[ $mode == "swift" ]]; then
  swift post ${bucket_name} --read-acl ".r:*,.rlistings"
  swift post "${bucket_name}_segments" --read-acl ".r:*,.rlistings"
fi

clean_temp_dir
echo ""
echo "Upload ready"
exit 0