submit

#!/bin/bash

set -e

irods_get() {
    local srcDataObj="$1"
    local destLocalDir="$2"

    # get a file from iRODS (overwrite existing and display progress)
    iget -f -P "$srcDataObj" "$destLocalDir"
}

# are we on OSG Connect?
if [ -e /stash -o -e /stash2 ]; then

    HNAME=`hostname -f`
    if [ "x$HNAME" != "xlogin03.osgconnect.net" ]; then
        echo "This workflow has to be submitted from login03.osgconnect.net"
        exit 1
    fi

    # needed for stashcp to be picked up the site catalog for the local site
    module load stashcache
    #module load pegasus
    export PATH=/cvmfs/oasis.opensciencegrid.org/osg/projects/pegasus/rhel7/4.9.0dev/bin:$PATH

    # make sure we also have access to the AMQP lib
    export PYTHONPATH="$PYTHONPATH:/usr/lib/python2.7/site-packages"

    export RUN_DIR=/local-scratch/$USER/workflows/$RUN_ID
    COMPUTE_SITE="osg-condorpool"
    STAGING_SITE="osg-stash"
elif [ -n "$2" ]; then
    export RUN_DIR=$PWD/workflows/$RUN_ID
    COMPUTE_SITE="generic-condorpool"
    STAGING_SITE="irods"
else
    export RUN_DIR=$PWD/workflows/$RUN_ID
    COMPUTE_SITE="generic-condorpool"
    STAGING_SITE="local"
fi

TOPDIR=`pwd`

JOB_COUNT=$1
if [ "x$JOB_COUNT" = "x" ]; then
    echo "Please specify the number of jobs. For example:"
    echo "  ./kinc 100"
    exit 1
fi

###
### copy the GEM from iRODS if a second script argument exists
###
gem_path="$2"
if [ -n "$gem_path" ]; then
    gem=$(basename $gem_path)
    # remove the yeast test data
    rm -f task-files/Yeast-GEM.txt
    rm -f task-files/Yeast-GEM.tar.gz
    # try to fetch the GEM file
    irods_get "$gem_path" task-files/
    if [ $? -ne 0 ]; then
        echo "Error: iRODS get of $gem_path failed."
        exit 1
    fi
    tar zxf task-files/$gem -C task-files/
fi

export RUN_ID=kinc-`date +'%s'`

#export RUN_DIR=/local-scratch/$USER/workflows/$RUN_ID

#generate sites.xml using the variables in this submit script

if [ "x$2" != "x" ]; then
    export irods_path=$(dirname $2)
fi
envsubst < "sites.xml.template" > "sites.xml"

# generate the dax
PYTHONPATH=`pegasus-config --python` ./tools/dax-generator $RUN_ID $RUN_DIR $JOB_COUNT

# plan and submit the  workflow.  Use irods as staging and output site if path is provided

if [ -n "$gem_path" ]; then

    pegasus-plan \
        --conf pegasus.conf \
        --cluster horizontal \
        --relative-dir $RUN_ID \
        --sites $COMPUTE_SITE \
        --staging-site $STAGING_SITE \
        --output-site irods \
        --dir $RUN_DIR \
        --dax dax.xml \
        --submit

else 
    
    pegasus-plan \
        --conf pegasus.conf \
        --cluster horizontal \
        --relative-dir $RUN_ID \
        --sites $COMPUTE_SITE \
        --staging-site $STAGING_SITE \
        --output-site local \
        --dir $RUN_DIR \
        --dax dax.xml \
        --submit
fi