Merge pull request #69 from cmeesters/fixes

Fixes

handout/README.txt

@@ -0,0 +1,41 @@
+Copyright (c) 2024, Snakemake Teaching Alliance + Christian Meester & JGU Mainz 
+   All rights reserved. 
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:     
+* Redistributions of source code must retain the above copyright notice, 
+  this list of conditions and the following disclaimer.     
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+* Neither the name of Christian Meesters or the JGU Mainz nor the names of 
+  its contributors may be used to endorse or promote products derived
+  from this software without specific prior written permission. 
+
+SAMPLE SCRIPTS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CHRISTIAN MEESTERS OR THE JGU 
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------
+
+Please note:
+
+* This handout does not display material to be used as a reference.
+* In particular, the recepient has been informed that people new
+  to Snakemake and HPC should partake this course themselves as (some) 
+  material can and will be outdated, eventually.
+* This is a handout release: Slides are merged and some (graphical) 
+  content has been dropped.
+* Sample scripts and (where applicable) solutions are provided. The above 
+  notice applies.
+
+
+Please direct suggestion for improvements to https://github.com/cmeesters/snakemake-hpc-teaching-material/issues .

pack_release.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+import os
+import shutil
+import subprocess
+import shlex
+import zipfile
+
+def run_pdflatex(master):
+    pwd = os.getcwd()
+    if os.sep in master: # probably a path
+       basename = os.path.basename(master)
+       dirname  = os.path.dirname(master)
+       os.chdir(dirname)
+       master = basename
+    call = r'pdflatex -synctex=1 -interaction=nonstopmode "\def\ishandout{1} \input{' + master + r'}"'
+    subprocess.call(shlex.split(call))
+    os.chdir(pwd)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', required = True,
+                        help = "indicate the handout version, e.g. edition")
+    parser.add_argument('--master-tex', required = True,
+                        help = "indicate a TeX master document")
+    parser.add_argument('--sample-directory', required = True,
+                        help = "path to directory with script files (cloze and solution")
+    args = parser.parse_args()
+
+    master = args.master_tex
+
+    print(f"Typesetting Handout Version for '{master}'")
+    #run_pdflatex(master)
+
+    handout_version = os.path.splitext(master)[0] + '.pdf'
+    final_place     = os.path.basename(handout_version)
+
+    file_list=[(handout_version, final_place),
+               ('handout/README.txt', 'README.txt')]
+    opj         = os.path.join
+    for root, dirs, files in os.walk(args.sample_directory):
+        for fname in files:
+            # restrict to sample files, no helper files
+            if 'copy' in fname or 'README' in fname: continue
+            file_list.append((opj(root,fname), opj(*root.split('/')[1:],fname)))
+
+    buildzipfname = 'snakemake_intro_%s.zip' % args.version
+    z = zipfile.ZipFile(buildzipfname, 'w', compression = zipfile.ZIP_DEFLATED)
+    for item in file_list:
+        z.write(item[0], item[1])
+    z.close()
+

setup/copy_script_mogonII.sh

@@ -6,10 +6,12 @@ CLUSTER_ALIAS="mogon_nox" # nox stands for "no X11 forwarding"
 BASEPATH="/lustre/project/hpckurs/workflows" # repeated used
 
 # creating remote directory:
-ssh mogon_nox "mkdir -p ${BASEPATH}"
+ssh ${CLUSTER_ALIAS} "mkdir -p ${BASEPATH}"
 
-scp condarc "${CLUSTER_ALIAS}:/lustre/project/hpckurs/workflows/condarc"
-scp get_tutorial.sh "${CLUSTER_ALIAS}:/lustre/project/hpckurs/workflows/get_tutorial.sh"
+scp condarc "${CLUSTER_ALIAS}:${BASEPATH}/condarc"
+scp get_tutorial.sh "${CLUSTER_ALIAS}:${BASEPATH}/get_tutorial.sh"
+scp install_micromamba.sh "${CLUSTER_ALIAS}:${BASEPATH}/install_micromamba.sh"
+scp environment.yaml "${CLUSTER_ALIAS}:${BASEPATH}/environment.yaml"
 
-rsync -rtlv --chmod=D755 "tutorial" "${CLUSTER_ALIAS}:/lustre/project/hpckurs/workflows"
+rsync -rtlv --chmod=D755 "tutorial" "${CLUSTER_ALIAS}:${BASEPATH}"
 rsync -rtlv --chmod=D755 "solutions" "${CLUSTER_ALIAS}:/lustre/project/hpckurs"

setup/environment.yaml

@@ -0,0 +1,18 @@
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - snakemake-minimal >=8.4.4
+  - snakemake-executor-plugin-slurm  
+  - snakemake-storage-plugin-fs
+  - jinja2
+  - matplotlib
+  - graphviz
+  - bcftools =1.19
+  - samtools =1.19.2
+  - bwa =0.7.17
+    #  - pysam =0.22
+    #  at the time of writing - 7. Feb 24 - pysam will require
+    #  a lower python version than snakemake, install pysam
+    #  using pip
+  - pygments

setup/install_micromamba.sh

@@ -0,0 +1 @@
+"${SHELL}" <(curl -L micro.mamba.pm/install.sh)

setup/tutorial/01_Snakefile

@@ -1,3 +1,9 @@
+# This is our first "rule" - it
+# serves as a template to proceed.
+#
+# All other templates are in the same
+# tutorial folder.
+
 rule bwa_map:
     input:
         "data/genome.fa",

slides/Snakemake_HPC_Creators.tex

@@ -2,6 +2,12 @@
 %       Snakemake-Intro for Workflow Creators for HPC Users       %
 %-----------------------------------------------------------------%
 
+% this code will compile the document as handout with
+% $ pdflatex -synctex=1 -interaction=nonstopmode "\def\ishandout{1} \input{Snakemake_HPC_Creators.tex}"
+\ifdefined\ishandout
+  \PassOptionsToClass{handout}{beamer}
+\fi
+
 \documentclass[english,xcolor=pdftex,dvipsnames]{beamer} 
 
 % to typeset only a few slide sets, set them here during development
@@ -74,7 +80,10 @@
 %\include{creators/Uploading_Workflows}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\include{common/Workflow_Parameterization_for_HPC}
+\include{creators/Workflow_Parameterization_for_HPC}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\include{common/Software_Provisioning_with_Snakemake}
 
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
 \include{common/using_wrappers}

slides/Snakemake_HPC_Users.tex

@@ -2,6 +2,13 @@
 %       Snakemake-Intro for HPC Users                   %
 %-------------------------------------------------------%
 
+
+% this code will compile the document as handout with
+% $ pdflatex -synctex=1 -interaction=nonstopmode "\def\ishandout{1} \input{Snakemake_HPC_Users.tex}"
+\ifdefined\ishandout
+\PassOptionsToClass{handout}{beamer}
+\fi
+
 \documentclass[english,xcolor=pdftex,dvipsnames]{beamer} 
 
 % to typeset only a few slide sets, set them here during development

slides/common/Hello_World_HPC_MogonII.tex

@@ -19,7 +19,7 @@
 				Save the script as \texttt{hello\_world.sh} and submit it with the following statement:
 			\end{task}
 			\begin{lstlisting}[language=Bash, style=Shell, basicstyle=\footnotesize]
-				$ sbatch hello_world.sh
+$ sbatch hello_world.sh
 			\end{lstlisting}
 		\end{onlyenv}
 		\begin{onlyenv}<2>

slides/common/Reports.tex

@@ -89,7 +89,7 @@ \subsection{Benchmarking}
   \end{itemize}
   \pause
   \begin{warning}
-  	We conclude: \Snakemake{}'s benchmarking capabilities are limited!
+  	We conclude: \Snakemake's benchmarking capabilities are limited, but a reasonable way to get basic benchmarks.
   \end{warning}
 \end{frame}
 
@@ -106,12 +106,55 @@ \subsection{Reporting}
   This will generate a file called ``\altverb{report.html}'', which you can visualize with a browser.
 \end{frame} 
 
+
 %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-\begin{frame}
-  \frametitle{\Snakemake{} Reports and Cluster Jobs}
-  \begin{alertblock}{Why reports about Cluster Jobs can be misleading.}
-    When reporting about a cluster or cloud job, \Snakemake{} shepherd job on the login-node will measure the wall time from submit time to the finish time, not the executing job(s).\newline
-    This means: Times can be greatly exaggerated!
-  \end{alertblock}
-\end{frame} 
+\begin{frame}[fragile]
+  \frametitle{\Snakemake{} Reports - adding Output}
+  \begin{docs}
+  	 Each output file that shall be part of the report has to be marked with the \altverb{report} flag, which optionally points to a caption in \lhref{https://docutils.sourceforge.io/docs/user/rst/quickstart.html}{restructured text format}.
+  \end{docs}
+  An example for our workflow would be:
+  \begin{lstlisting}[language=Python,style=Python]
+rule plot_quals:
+    input:
+        "calls/all.vcf"
+    output:
+        @report("plots/quals.svg",@ 
+               @caption="report/qual.rst")@
+    script:
+        "scripts/plot-quals.py"
+  \end{lstlisting}
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}[fragile]
+  \frametitle{\HandsOn{Writing an Annotation}}
+  Let's write the file \altverb{report/qual.rst}! It shall contain our caption.
+  \pause
+  Our solution might(!) look like this:
+  \begin{lstlisting}
+Number of variations (deviations from reference) 
+per experimental record. 
+  \end{lstlisting}	
+\end{frame}
+
+%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
+\begin{frame}[fragile]
+	\frametitle{\HandsOn{Adding our Figure to the Report}}
+	Now add the highlighted lines:
+	\begin{lstlisting}[language=Python,style=Python]
+rule plot_quals:
+    input:
+	    "calls/all.vcf"
+	output:
+		@report("plots/quals.svg",@ 
+		@caption="report/qual.rst")@
+	script:
+		"scripts/plot-quals.py"
+	\end{lstlisting}
+    \begin{task}{Re-Run our report generator:}
+    	\altverb{snakemake --report}
+    \end{task}
+\end{frame}
+