slides/common/preamble.tex - resolved merge conflict

handout/README.txt

@@ -0,0 +1,41 @@
+Copyright (c) 2024, Snakemake Teaching Alliance + Christian Meester & JGU Mainz 
+   All rights reserved. 
+
+Redistribution and use in source and binary forms, with or without 
+modification, are permitted provided that the following conditions are met:     
+* Redistributions of source code must retain the above copyright notice, 
+  this list of conditions and the following disclaimer.     
+* Redistributions in binary form must reproduce the above copyright notice,
+  this list of conditions and the following disclaimer in the documentation
+  and/or other materials provided with the distribution.
+* Neither the name of Christian Meesters or the JGU Mainz nor the names of 
+  its contributors may be used to endorse or promote products derived
+  from this software without specific prior written permission. 
+
+SAMPLE SCRIPTS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 
+PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL CHRISTIAN MEESTERS OR THE JGU 
+BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 
+CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 
+SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 
+INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 
+WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 
+OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF 
+ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+----------------------
+
+Please note:
+
+* This handout does not display material to be used as a reference.
+* In particular, the recepient has been informed that people new
+  to Snakemake and HPC should partake this course themselves as (some) 
+  material can and will be outdated, eventually.
+* This is a handout release: Slides are merged and some (graphical) 
+  content has been dropped.
+* Sample scripts and (where applicable) solutions are provided. The above 
+  notice applies.
+
+
+Please direct suggestion for improvements to https://github.com/cmeesters/snakemake-hpc-teaching-material/issues .

pack_release.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+import os
+import shutil
+import subprocess
+import shlex
+import zipfile
+
+def run_pdflatex(master):
+    pwd = os.getcwd()
+    if os.sep in master: # probably a path
+       basename = os.path.basename(master)
+       dirname  = os.path.dirname(master)
+       os.chdir(dirname)
+       master = basename
+    call = r'pdflatex -synctex=1 -interaction=nonstopmode "\def\ishandout{1} \input{' + master + r'}"'
+    subprocess.call(shlex.split(call))
+    os.chdir(pwd)
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--version', required = True,
+                        help = "indicate the handout version, e.g. edition")
+    parser.add_argument('--master-tex', required = True,
+                        help = "indicate a TeX master document")
+    parser.add_argument('--sample-directory', required = True,
+                        help = "path to directory with script files (cloze and solution")
+    args = parser.parse_args()
+
+    master = args.master_tex
+
+    print(f"Typesetting Handout Version for '{master}'")
+    #run_pdflatex(master)
+
+    handout_version = os.path.splitext(master)[0] + '.pdf'
+    final_place     = os.path.basename(handout_version)
+
+    file_list=[(handout_version, final_place),
+               ('handout/README.txt', 'README.txt')]
+    opj         = os.path.join
+    for root, dirs, files in os.walk(args.sample_directory):
+        for fname in files:
+            # restrict to sample files, no helper files
+            if 'copy' in fname or 'README' in fname: continue
+            file_list.append((opj(root,fname), opj(*root.split('/')[1:],fname)))
+
+    buildzipfname = 'snakemake_intro_%s.zip' % args.version
+    z = zipfile.ZipFile(buildzipfname, 'w', compression = zipfile.ZIP_DEFLATED)
+    for item in file_list:
+        z.write(item[0], item[1])
+    z.close()
+

setup/README.txt

@@ -0,0 +1,14 @@
+This README file describes the setup for a "Creator" course
+
+1. copy the 'condarc' file and the 'get_tutorial.sh' script
+   to a file space which is accessible for every participant.
+2. enter this directory as the 'pathtosetup' parameter in
+   'slides/config/config.dat'
+3. copy the 'tutorial' directory to a file space (directory)
+   accessible for every participant
+4. enter this directory as the 'pathtoclozure' parameter in
+   'slides/config/config.dat'
+5. finally repeat this for the directory 'solutions',
+   the configuration parameter is 'pathtosolutions'
+
+

setup/copy_script_mogonII.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+# This script is used to teach on the cluster "Mogon II"
+
+CLUSTER_ALIAS="mogon_nox" # nox stands for "no X11 forwarding"
+BASEPATH="/lustre/project/hpckurs/workflows" # repeated used
+
+# creating remote directory:
+ssh ${CLUSTER_ALIAS} "mkdir -p ${BASEPATH}"
+
+scp condarc "${CLUSTER_ALIAS}:${BASEPATH}/condarc"
+scp get_tutorial.sh "${CLUSTER_ALIAS}:${BASEPATH}/get_tutorial.sh"
+scp install_micromamba.sh "${CLUSTER_ALIAS}:${BASEPATH}/install_micromamba.sh"
+scp environment.yaml "${CLUSTER_ALIAS}:${BASEPATH}/environment.yaml"
+
+rsync -rtlv --chmod=D755 "tutorial" "${CLUSTER_ALIAS}:${BASEPATH}"
+rsync -rtlv --chmod=D755 "solutions" "${CLUSTER_ALIAS}:/lustre/project/hpckurs"

setup/copy_script_template.sh

@@ -0,0 +1,12 @@
+#!/bin/bash
+
+# Please note: if your ssh-configuration contains "ForwardX11 yes" rsync
+# might not return. In this case, use a config without X11 forwarding.
+
+CLUSTER_ALIAS=<cluster_name>
+
+scp condarc "${CLUSTER_ALIAS}:/path/to/setup/dir"
+scp get_tutorial.sh "${CLUSTER_ALIAS}:/path/to/setup/dir"
+
+rsync -rtlv --chmod=D755 "tutorial" "${CLUSTER_ALIAS}:/path/to/tutorial"
+rsync -rtlv --chmod=D755 "solutions" "${CLUSTER_ALIAS}:/path/to/tutorial"

setup/environment.yaml

@@ -0,0 +1,18 @@
+channels:
+  - conda-forge
+  - bioconda
+dependencies:
+  - snakemake-minimal >=8.4.4
+  - snakemake-executor-plugin-slurm  
+  - snakemake-storage-plugin-fs
+  - jinja2
+  - matplotlib
+  - graphviz
+  - bcftools =1.19
+  - samtools =1.19.2
+  - bwa =0.7.17
+    #  - pysam =0.22
+    #  at the time of writing - 7. Feb 24 - pysam will require
+    #  a lower python version than snakemake, install pysam
+    #  using pip
+  - pygments

setup/install_micromamba.sh

@@ -0,0 +1 @@
+"${SHELL}" <(curl -L micro.mamba.pm/install.sh)

setup/solutions/02_Snakefile

@@ -0,0 +1,10 @@
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/A.fastq"
+    output:
+        "mapped_reads/A.bam"
+    shell:
+        "bwa mem {input}"
+        " | samtools view -Sb - >"
+        " {output}"

setup/solutions/03_Snakefile

@@ -0,0 +1,10 @@
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/{sample}.fastq"
+    output:
+        "mapped_reads/{sample}.bam"
+    shell:
+        "bwa mem {input}"
+        " | samtools view -Sb - >"
+        " {output}"

setup/solutions/04_Snakefile

@@ -0,0 +1,20 @@
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/A.fastq"
+    output:
+        "mapped_reads/A.bam"
+    shell:
+        "bwa mem data/genome.fa"
+        " data/samples/A.fastq"
+        " | samtools view -Sb - >"
+        " mapped_reads/A.bam"
+
+rule samtools_sort:
+    input:
+        "mapped_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam"
+    shell:
+        "samtools sort -T sorted_reads/{wildcards.sample} "
+        "-O bam {input} > {output}"

setup/solutions/05_Snakefile

@@ -0,0 +1,49 @@
+# our samples are pre-configured
+SAMPLES = ["A", "B"]
+
+rule all:
+    input:
+        "calls/all.vcf"
+
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/{sample}.fastq"
+    output:
+        "mapped_reads/{sample}.bam"
+    shell:
+        "bwa mem {input} | samtools view -Sb - > {output}"
+
+
+rule samtools_sort:
+    input:
+        "mapped_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam"
+    shell:
+        "samtools sort -T sorted_reads/{wildcards.sample} "
+        "-O bam {input} > {output}"
+
+
+rule samtools_index:
+    input:
+        "sorted_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam.bai"
+    shell:
+        "samtools index {input}"
+
+
+rule bcftools_call:
+    input:
+        fa="data/genome.fa",
+        bam=expand("sorted_reads/{sample}.bam", sample=SAMPLES),
+        bai=expand("sorted_reads/{sample}.bam.bai", sample=SAMPLES)
+    output:
+        "calls/all.vcf"
+    shell:
+        "bcftools mpileup -f {input.fa} {input.bam} | "
+        "bcftools call -mv - > {output}"
+
+
+

setup/solutions/06_Snakefile

@@ -0,0 +1,57 @@
+# our samples are pre-configured
+SAMPLES = ["A", "B"]
+
+rule all:
+    input:
+        "plots/quals.svg",
+        "calls/all.vcf"
+
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/{sample}.fastq"
+    output:
+        "mapped_reads/{sample}.bam"
+    shell:
+        "bwa mem {input} | samtools view -Sb - > {output}"
+
+
+rule samtools_sort:
+    input:
+        "mapped_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam"
+    shell:
+        "samtools sort -T sorted_reads/{wildcards.sample} "
+        "-O bam {input} > {output}"
+
+
+rule samtools_index:
+    input:
+        "sorted_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam.bai"
+    shell:
+        "samtools index {input}"
+
+
+rule bcftools_call:
+    input:
+        fa="data/genome.fa",
+        bam=expand("sorted_reads/{sample}.bam", sample=SAMPLES),
+        bai=expand("sorted_reads/{sample}.bam.bai", sample=SAMPLES)
+    output:
+        "calls/all.vcf"
+    shell:
+        "bcftools mpileup -f {input.fa} {input.bam} | "
+        "bcftools call -mv - > {output}"
+
+rule plot_quals:
+    input:
+       "calls/all.vcf"
+    output:
+       "plots/quals.svg"
+    script:
+       "scripts/plot-quals.py"
+
+

setup/solutions/07_Snakefile

@@ -0,0 +1,54 @@
+SAMPLES = ['A', 'B']
+
+rule all:
+    input:
+        "plots/quals.svg",
+        "calls/all.vcf"
+
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/{sample}.fastq"
+    output:
+        "mapped_reads/{sample}.bam"
+    shell:
+        "bwa mem {input} | "
+        "samtools view -Sb - > {output}"
+
+rule samtools_sort:
+    input:
+        "mapped_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam"
+    shell:
+        "samtools sort -T sorted_reads/{wildcards.sample} "
+        "-O bam {input} > {output}"
+
+
+rule samtools_index:
+    input:
+        "sorted_reads/{sample}.bam"
+    output:
+        "sorted_reads/{sample}.bam.bai"
+    shell:
+        "samtools index {input}"
+
+rule bcftools_call:
+    input:
+        fa="data/genome.fa",
+        bam=expand("sorted_reads/{sample}.bam", sample=config["samples"]),
+        bai=expand("sorted_reads/{sample}.bam.bai", sample=config["samples"])
+    output:
+        "calls/all.vcf"
+    shell:
+        "bcftools mpileup -f {input.fa} {input.bam} | "
+        "bcftools call -mv - > {output}"
+
+
+rule plot_quals:
+    input:
+        "calls/all.vcf"
+    output:
+        "plots/quals.svg"
+    script:
+        "scripts/plot-quals.py"

setup/solutions/scripts/plot-quals.py

@@ -0,0 +1,10 @@
+import matplotlib
+matplotlib.use("Agg")
+import matplotlib.pyplot as plt
+from pysam import VariantFile
+
+quals = [record.qual for record 
+         in VariantFile(snakemake.input[0])]
+
+plt.hist(quals)
+plt.savefig(snakemake.output[0])

setup/tutorial/01_Snakefile

@@ -0,0 +1,17 @@
+# This is our first "rule" - it
+# serves as a template to proceed.
+#
+# All other templates are in the same
+# tutorial folder.
+
+rule bwa_map:
+    input:
+        "data/genome.fa",
+        "data/samples/A.fastq"
+    output:
+        "mapped_reads/A.bam"
+    shell:
+        "bwa mem data/genome.fa"
+        " data/samples/A.fastq"
+        " | samtools view -Sb - >"
+        " mapped_reads/A.bam"