feat(Google-Batch): Configurable mounting point via config option google.batch.mountRoot

This commit adds the ability to configure the root of the mounting point for gcsfuse such that you could specify where the data is mounted. It is controlled by the config item google.batch.mountRoot with a default of /mnt/disks

Signed-off-by: adamrtalbot <[email protected]>

Author: adamrtalbot

docs/reference/config.md

@@ -878,6 +878,11 @@ The following settings are available for Google Cloud Batch:
 : Max number of execution attempts of a job interrupted by a Compute Engine Spot reclaim event (default: `0`).
 : See also: `google.batch.autoRetryExitCodes`
 
+`google.batch.mountRoot`
+: :::{versionadded} 25.05.0-edge
+  :::
+: The root directory for GCS mounting in the container (default: `/mnt/disks`).
+
 `google.batch.network`
 : The URL of an existing network resource to which the VM will be attached.
 

plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchExecutor.groovy

@@ -168,7 +168,10 @@ class GoogleBatchExecutor extends Executor implements ExtensionPoint, TaskArrayE
     String getArrayWorkDir(TaskHandler handler) {
         return isFusionEnabled() || isWorkDirDefaultFS()
             ? TaskArrayExecutor.super.getArrayWorkDir(handler)
-            : containerMountPath(handler.task.workDir as CloudStoragePath)
+            : containerMountPath(
+                handler.task.workDir as CloudStoragePath, 
+                config.getMountRoot()
+            )
     }
 
     @Override

plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchScriptLauncher.groovy

@@ -43,8 +43,6 @@ import nextflow.util.TestOnly
 @CompileStatic
 class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatchLauncherSpec {
 
-    private static final String MOUNT_ROOT = '/mnt/disks'
-
     private BatchConfig config
     private CloudStoragePath remoteWorkDir
     private Path remoteBinDir
@@ -55,8 +53,11 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
     @TestOnly
     protected GoogleBatchScriptLauncher() {}
 
-    GoogleBatchScriptLauncher(TaskBean bean, Path remoteBinDir) {
+    GoogleBatchScriptLauncher(TaskBean bean, Path remoteBinDir, BatchConfig config) {
         super(bean)
+        // Initialize config first before using any methods that depend on it
+        this.config = config
+        
         // keep track the google storage work dir
         this.remoteWorkDir = (CloudStoragePath) bean.workDir
         this.remoteBinDir = toContainerMount(remoteBinDir)
@@ -116,7 +117,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
         if( path instanceof CloudStoragePath ) {
             buckets.add(path.bucket())
             pathTrie.add( (parent ? "/${path.bucket()}${path.parent}" : "/${path.bucket()}${path}").toString() )
-            final containerMount = containerMountPath(path)
+            final containerMount = containerMountPath(path, config.getMountRoot())
             log.trace "Path ${FilesEx.toUriString(path)} to container mount: $containerMount"
             return Paths.get(containerMount)
         }
@@ -135,15 +136,17 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
     @Override
     List<String> getContainerMounts() {
         final result = new ArrayList(10)
+        final mountRoot = config.getMountRoot()
         for( String it : pathTrie.longest() ) {
-            result.add( "${MOUNT_ROOT}${it}:${MOUNT_ROOT}${it}:rw".toString() )
+            result.add( "${mountRoot}${it}:${mountRoot}${it}:rw".toString() )
         }
         return result
     }
 
     @Override
     List<Volume> getVolumes() {
         final result = new ArrayList(10)
+        final mountRoot = config.getMountRoot()
         for( String it : buckets ) {
             final mountOptions = new LinkedList<String>()
             if( config && config.gcsfuseOptions )
@@ -157,7 +160,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
                         GCS.newBuilder()
                             .setRemotePath(it)
                     )
-                    .setMountPath( "${MOUNT_ROOT}/${it}".toString() )
+                    .setMountPath( "${mountRoot}/${it}".toString() )
                     .addAllMountOptions( mountOptions )
                     .build()
             )
@@ -184,11 +187,6 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
         return remoteWorkDir.resolve(TaskRun.CMD_INFILE)
     }
 
-    GoogleBatchScriptLauncher withConfig(BatchConfig config) {
-        this.config = config
-        return this
-    }
-
     GoogleBatchScriptLauncher withIsArray(boolean value) {
         this.isArray = value
         return this
@@ -207,7 +205,7 @@ class GoogleBatchScriptLauncher extends BashWrapperBuilder implements GoogleBatc
         "trap \"{ cp ${TaskRun.CMD_LOG} ${workDir}/${TaskRun.CMD_LOG}; }\" ERR; /bin/bash ${workDir}/${TaskRun.CMD_RUN} 2>&1 | tee ${TaskRun.CMD_LOG}"
     }
 
-    static String containerMountPath(CloudStoragePath path) {
-        return "$MOUNT_ROOT/${path.bucket()}${path}"
+    static String containerMountPath(CloudStoragePath path, String mountRoot) {
+        return "${mountRoot}/${path.bucket()}${path}"
     }
 }

plugins/nf-google/src/main/nextflow/cloud/google/batch/GoogleBatchTaskHandler.groovy

@@ -37,6 +37,7 @@ import groovy.transform.CompileStatic
 import groovy.transform.PackageScope
 import groovy.util.logging.Slf4j
 import nextflow.cloud.google.batch.client.BatchClient
+import nextflow.cloud.google.batch.client.BatchConfig
 import nextflow.cloud.types.CloudMachineInfo
 import nextflow.cloud.types.PriceModel
 import nextflow.exception.ProcessException
@@ -148,8 +149,8 @@ class GoogleBatchTaskHandler extends TaskHandler implements FusionAwareTask {
         }
         else {
             final taskBean = task.toTaskBean()
-            return new GoogleBatchScriptLauncher(taskBean, executor.remoteBinDir)
-                .withConfig(executor.config)
+            final config = executor.config ?: new BatchConfig()
+            return new GoogleBatchScriptLauncher(taskBean, executor.remoteBinDir, config)
                 .withIsArray(task.isArray())
         }
     }

plugins/nf-google/src/main/nextflow/cloud/google/batch/client/BatchConfig.groovy

@@ -37,6 +37,8 @@ class BatchConfig {
 
     static final private List<String> DEFAULT_GCSFUSE_OPTS = List.<String>of('-o rw', '-implicit-dirs')
 
+    static final private String DEFAULT_MOUNT_ROOT = '/mnt/disks'
+
     private GoogleOpts googleOpts
     private GoogleCredentials credentials
     private List<String> allowedLocations
@@ -55,6 +57,7 @@ class BatchConfig {
     private BatchRetryConfig retryConfig
     private List<Integer> autoRetryExitCodes
     private List<String> gcsfuseOptions
+    private String mountRoot
 
     GoogleOpts getGoogleOpts() { return googleOpts }
     GoogleCredentials getCredentials() { return credentials }
@@ -74,6 +77,7 @@ class BatchConfig {
     BatchRetryConfig getRetryConfig() { retryConfig }
     List<Integer> getAutoRetryExitCodes() { autoRetryExitCodes }
     List<String> getGcsfuseOptions() { gcsfuseOptions }
+    String getMountRoot() { mountRoot }
 
     static BatchConfig create(Session session) {
         final result = new BatchConfig()
@@ -95,6 +99,7 @@ class BatchConfig {
         result.retryConfig = new BatchRetryConfig( session.config.navigate('google.batch.retryPolicy') as Map ?: Map.of() )
         result.autoRetryExitCodes = session.config.navigate('google.batch.autoRetryExitCodes', DEFAULT_RETRY_LIST) as List<Integer>
         result.gcsfuseOptions = session.config.navigate('google.batch.gcsfuseOptions', DEFAULT_GCSFUSE_OPTS) as List<String>
+        result.mountRoot = session.config.navigate('google.batch.mountRoot', DEFAULT_MOUNT_ROOT) as String
         return result
     }