[Demo][WIP] Storage mounting

examples/storage_mount_demo.yaml

@@ -0,0 +1,29 @@
+name: storage-demo
+
+resources:
+  cloud: aws
+  instance_type: m5.2xlarge
+
+num_nodes: 2
+
+file_mounts:
+  /sharedfs:
+    name: romil-fs
+    source: ~/tmp # Empty dir for MVP, this will not be required if mode==MOUNT
+    persistent: True
+    mode: 'MOUNT'
+
+run: |
+  echo This is hostname: $(hostname)
+
+  for ((i=1;i<=100;i++)); 
+    do 
+      # Run example write
+      echo ${i}
+      echo File ${i}, My hostname: $(hostname) > /sharedfs/$(hostname)_${i}.txt
+      sleep 1
+    done
+
+
+  # ls and exit
+  ls -la /sharedfs

sky/data/storage.py

@@ -25,6 +25,11 @@ class StorageType(enum.Enum):
     AZURE = 'AZURE'
 
 
+class StorageMode(enum.Enum):
+    MOUNT = 'MOUNT'
+    COPY = 'COPY'
+
+
 class AbstractStore:
     """AbstractStore abstracts away the different storage types exposed by
     different clouds.
@@ -148,7 +153,8 @@ def __init__(self,
                  name: str,
                  source: Path,
                  stores: Optional[Dict[StorageType, AbstractStore]] = None,
-                 persistent: bool = True):
+                 persistent: bool = True,
+                 mode: StorageMode = StorageMode.MOUNT):
         """Initializes a Storage object.
 
         Three fields are required: the name of the storage, the source
@@ -163,10 +169,13 @@ def __init__(self,
             need to be absolute.
           stores: Optional; Specify pre-initialized stores (S3Store, GcsStore).
           persistent: bool; Whether to persist across sky launches.
+          mode: StorageMode; Specify how the storage object is manifested on
+            the remote VM. Can be either MOUNT or COPY.
         """
         self.name = name
         self.source = source
         self.persistent = persistent
+        self.mode = mode
 
         scheme = urllib.parse.urlsplit(self.source).scheme
         is_bucket_url = False
@@ -368,7 +377,7 @@ def sync_local_dir(self) -> None:
         increase parallelism, modify max_concurrent_requests in your aws config
         file (Default path: ~/.aws/config).
         """
-        sync_command = f'aws s3 sync {self.source} s3://{self.name}/ --delete'
+        sync_command = f'aws s3 sync {self.source} s3://{self.name}/ --delete'  # TODO(romilb): Delete is problematic if task writes to s3 and the user runs this again...
         logger.info(f'Executing: {sync_command}')
         with subprocess.Popen(sync_command.split(' '),
                               stderr=subprocess.PIPE) as process:

sky/task.py

@@ -10,6 +10,7 @@
 from sky import clouds
 from sky import resources as resources_lib
 from sky.data import storage as storage_lib
+from sky.data.storage import StorageMode
 
 Resources = resources_lib.Resources
 # A lambda generating commands (node rank_i, node addrs -> cmd_i).
@@ -247,13 +248,15 @@ def from_yaml(yaml_path):
             name = storage.get('name')
             source = storage.get('source')
             force_stores = storage.get('force_stores')
+            mode = storage.get('mode')
             assert name and source, \
                    'Storage Object needs name and source path specified.'
             persistent = True if storage.get(
                 'persistent') is None else storage['persistent']
             storage_obj = storage_lib.Storage(name=name,
                                               source=source,
-                                              persistent=persistent)
+                                              persistent=persistent,
+                                              mode=StorageMode(mode))
             if force_stores is not None:
                 assert set(force_stores) <= {'s3', 'gcs', 'azure_blob'}
                 for cloud_type in force_stores:
@@ -432,9 +435,16 @@ def add_storage_mounts(self) -> None:
             storage_type = storage_plans[store]
             if storage_type is storage_lib.StorageType.S3:
                 # TODO: allow for Storage mounting of different clouds
-                self.update_file_mounts({
-                    mnt_path: 's3://' + store.name,
-                })
+                if store.mode == StorageMode.MOUNT:
+                    self.setup = (
+                        '(sudo wget https://github.com/kahing/goofys/releases/latest/download/goofys'
+                        ' -O /usr/local/bin/goofys && sudo chmod +x /usr/local/bin/goofys && '
+                        f' sudo mkdir -p {mnt_path} && sudo chmod 777 {mnt_path} && goofys --stat-cache-ttl 3s --type-cache-ttl 3s {store.name} {mnt_path}'
+                        f'); {self.setup or "true"}')
+                else:
+                    self.update_file_mounts({
+                        mnt_path: 's3://' + store.name,
+                    })
             elif storage_type is storage_lib.StorageType.GCS:
                 # Remember to run `gcloud auth application-default login`
                 self.setup = (