ENH: pipeline now ready to be containerised

main.nf

@@ -17,9 +17,9 @@ validate(params)
 include { singularityPreflight } from "$projectDir/modules/singularity"
 // If Singularity is used as the container engine and not showing help message, do preflight check to prevent parallel pull issues
 // Related issue: https://github.com/nextflow-io/nextflow/issues/1210
-// if (workflow.containerEngine == 'singularity') {
-//     singularityPreflight(workflow.container, params.singularity_cachedir)
-// }
+if (workflow.containerEngine == 'singularity') {
+    singularityPreflight(workflow.container, params.singularity_cachedir)
+}
 
 
 workflow {

modules/singularity.nf

@@ -0,0 +1,48 @@
+// Courtesy of Harry Hung, original script can be found at https://github.com/sanger-bentley-group/gps-pipeline/blob/master/modules/singularity.nf
+// Check if Singularity images are already pulled, otherwise pull non-existing images one by one
+void singularityPreflight(LinkedHashMap workflowContainer, String singularityCacheDir) {
+    log.info("Checking if all the Singularity images are available at ${singularityCacheDir}\n")
+
+    // Get names of all images
+    containers = workflowContainer.collect { it.value } as Set
+
+    // Create the directory for saving images if not yet existed
+    File cacheDir = new File(singularityCacheDir)
+    cacheDir.exists() || cacheDir.mkdirs()
+
+    // Get images that needs to be downloaded
+    toDownload = [] as Set
+    containers.each { container ->
+        targetName = container.replace(':', '-').replace('/', '-') + '.img'
+        targetFile = new File (singularityCacheDir + File.separator + targetName)
+        if (!targetFile.exists()) {
+            toDownload.add([container, targetName])
+        }
+    }
+
+    // Download all the images that do not exist yet
+    toDownload.each { container, targetName ->
+        log.info("${container} is not found. Pulling now...")
+        process = "singularity pull --dir ${singularityCacheDir} ${targetName} docker://${container}".execute()
+        process.waitFor()
+
+        if (process.exitValue()) {
+            def errorMessage = new BufferedReader(new InputStreamReader(process.getErrorStream())).getText()
+
+            log.info(
+                """
+                |Singularity Error Messages:
+                |${errorMessage}
+                | 
+                |${container} cannot be pulled successfully. Resolve the above error and re-run the pipeline.
+                |
+                """.stripMargin()
+            )
+            System.exit(1)
+        }
+
+        log.info("${container} is pulled and saved as ${targetName}\n")
+    }
+
+    log.info("All images are ready. The workflow will resume.\n")
+}

modules/validate.nf

@@ -4,6 +4,7 @@
 
 validParams = [
     with_match_normal: 'boolean',
+    singularity_cachedir: 'path',
     sample_paths: 'path',
     concordance_threshold: 'number',
     contamination_threshold_samples: 'number',

sanger_lsf.config

@@ -7,12 +7,13 @@ params {
     max_memory = 128.GB
     max_cpus = 64
     max_time = 48.h
+    singularity_cachedir = "${projectDir}/singularity"
 }
 
 
 singularity {
     enabled = true
-    cacheDir = "${projectDir}/singularity"
+    cacheDir = params.singularity_cachedir
     autoMounts = true
 }