rclone-ls-workaround

#!/bin/bash
#
# "rclone ls" with swift access lists only part of the objects if
# there are a lot of them in a bucket (probably around 100000).
#
# This is a workaround script for puhti.
#
# This uses s3 access to list the objects. As segmented objects are reported
# as size 0 objects, script uses "rclone lsl" to get the correct size.
# You must have a rclone configuration with remote s3allas.
# With option -d the script uses "rclone lsl" for objects ls'ed smaller than 10k.
#
# Note: simply running "rclone lsl s3allas:bucket" for a lot of objects
#       would be unusable slow. Also if there are a lot of size 0 objects
#       (or size<10k) or segmented objects, this script will be very slow.

FIX_LEN=10
FIX_STR='        0 '

if [ "$1" == '-d' ]; then
	FIX_LEN=5
	FIX_STR='     '
	shift
fi

if [ $# -ne 1 ]; then
	echo "Usage: $0 bucket_name"
	exit 1
fi

rclone ls s3allas:"$1" | while IFS= read -r line
do
	if [ "${line:0:$FIX_LEN}" == "$FIX_STR" ]; then
		name=`echo $line|sed 's/ *[0-9]* \(.*\)/\1/'`
		rclone lsl s3allas:"$1"/"$name" | awk '{printf "%9s", $1}'
		echo " $name"
	else
		echo "$line"
	fi
done