cheyenneJobs.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {},
   "outputs": [],
   "source": [
    "import os, shutil\n",
    "import subprocess\n",
    "import glob"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Directories\n",
    "masterdir = '/glade/p/work/manab/ff/islandpark/'\n",
    "logdname = 'log'\n",
    "jobdname = 'joblists'\n",
    "pbsdname = 'pbsscripts'\n",
    "\n",
    "#FUSE\n",
    "fuseexe = '/glade/p/work/manab/ff/newfuse/fuse/bin/fuse.exe'\n",
    "configtpl = '/glade/p/work/manab/ff/islandpark/fm_sacsma.txt' #Main Config template\n",
    "inputInfoTpl = '/glade/p/work/manab/ff/islandpark/settings/template_input_info.txt'  #Input info template\n",
    "forcdir = '/glade/p/work/manab/ff/islandpark/inputnew'\n",
    "fusemode = 'run_def'   #Options\n",
    "simstarttime = '2017'\n",
    "simendtime = '2017'\n",
    "evalstarttime = '2017'\n",
    "evalendtime = '2017'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def concatDir(dir, subdir):\n",
    "    '''\n",
    "    Concatenates Master dir and another dname (directory name)\n",
    "    '''\n",
    "    newdir = dir + subdir\n",
    "    return(newdir)\n",
    "\n",
    "def purgeDir(folder):\n",
    "    '''\n",
    "    Purges contents of a directory\n",
    "    '''\n",
    "\n",
    "    for the_file in os.listdir(folder):\n",
    "        file_path = os.path.join(folder, the_file)\n",
    "        try:\n",
    "            if os.path.isfile(file_path):\n",
    "                os.unlink(file_path)\n",
    "        except Exception as e:\n",
    "            print(e)\n",
    "\n",
    "#def configFile(file):\n",
    "#    '''\n",
    "#    Writes config files for each ensemble run\n",
    "#    '''\n",
    "#    with open(file, \"rt\") as fin:\n",
    "#        with open('/glade/p/work/manab/ff/islandpark/fm_template2.txt', \"wt\") as fout:\n",
    "#            for line in fin:\n",
    "#                print(line)\n",
    "#                fout.write(line.replace('SIMSTART', simstarttime)\n",
    " \n",
    "def forcingFiles(folder):\n",
    "    '''\n",
    "    Find all forcing files for each input\n",
    "    '''\n",
    "    files = glob.glob(folder + '/*nc')\n",
    "    files = [x for x in files if \"elev_bands\" not in x]\n",
    "    return(files)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 169,
   "metadata": {},
   "outputs": [],
   "source": [
    "if __name__ == '__main__':\n",
    "    logdir = concatDir(masterdir, logdname)\n",
    "    jobdir = concatDir(masterdir, jobdname)\n",
    "    pbsdir = concatDir(masterdir, pbsdname)\n",
    "    \n",
    "    purgeDir(logdir)      #Empty, if not.\n",
    "    purgeDir(jobdir)\n",
    "    purgeDir(pbsdir)\n",
    "    \n",
    "    forcfiles = sorted(forcingFiles(forcdir))\n",
    "    \n",
    "    #Create input info files\n",
    "    out1 = []\n",
    "    out2 = []\n",
    "    out3 = []\n",
    "    runcoms = []\n",
    "    for count, value in enumerate(forcfiles):\n",
    "        outfilename1 = os.path.basename(value)     #001.nc\n",
    "        outfilename2 = outfilename1.split('.')[0]   #001    \n",
    "        outfile = os.path.join(os.path.split(inputInfoTpl)[0], outfilename2 + '_input_info.txt') \n",
    "        out1.append(outfilename1)\n",
    "        out2.append(outfilename2)\n",
    "        out3.append(outfile)\n",
    "        \n",
    "        runcomx = [fuseexe, configtpl, outfilename2, '001', fusemode]\n",
    "        runcomx = \" \".join(runcomx)\n",
    "        \n",
    "        runcoms.append(runcomx)\n",
    "\n",
    "        with open('/glade/p/work/manab/ff/islandpark/settings/template_input_info.txt', \"rt\") as fin:\n",
    "            with open(outfile, \"wt\") as fout:\n",
    "                for line in fin:\n",
    "                    fout.write(line.replace('FORCTEMP', outfilename1))\n",
    "    \n",
    "        #Create joblist\n",
    "        \n",
    "        \n",
    "        for x,y in enumerate(range(0, len(runcoms), 36)):\n",
    "            jobchunk = runcoms[y : y + 36]\n",
    "    \n",
    "            with open(os.path.join(jobdir, str(x)), \"wt\") as fout:\n",
    "                for item in jobchunk:\n",
    "                    fout.write(\"{}\\n\".format(item))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 001 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 002 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 003 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 004 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 005 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 006 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 007 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 008 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 009 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 010 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 011 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 012 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 013 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 014 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 015 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 016 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 017 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 018 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 019 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 020 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 021 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 022 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 023 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 024 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 025 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 026 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 027 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 028 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 029 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 030 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 031 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 032 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 033 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 034 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 035 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 036 001 run_def']\n",
      "['/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 037 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 038 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 039 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 040 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 041 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 042 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 043 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 044 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 045 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 046 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 047 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 048 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 049 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 050 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 051 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 052 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 053 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 054 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 055 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 056 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 057 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 058 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 059 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 060 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 061 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 062 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 063 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 064 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 065 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 066 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 067 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 068 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 069 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 070 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 071 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 072 001 run_def']\n",
      "['/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 073 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 074 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 075 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 076 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 077 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 078 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 079 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 080 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 081 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 082 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 083 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 084 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 085 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 086 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 087 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 088 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 089 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 090 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 091 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 092 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 093 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 094 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 095 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 096 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 097 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 098 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 099 001 run_def', '/glade/p/work/manab/ff/fuse/bin/fuse.exe /glade/p/work/manab/ff/islandpark/fm_template_workingbkp.txt 100 001 run_def']\n"
     ]
    }
   ],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python (SHARP)",
   "language": "python",
   "name": "sharp"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}