Localtomodal (#24)

* data transfer from local to modal

* Update readme.md

* Update readme.md

* minor changes

* removed old file

* added a script to upload a floor from local

* Update readme.md

* updated the volume name

src/modal_functions/volume/add_new_map.py

@@ -0,0 +1,28 @@
+import modal
+import os
+import logging
+
+logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
+
+volume = modal.Volume.from_name("NewVisiondata", create_if_missing=True)
+
+
+def process_upload(remote_path, folder_path):
+    with volume.batch_upload() as batch: 
+        try:
+            batch.put_directory(folder_path, remote_path)
+            logging.info(f"Successfully uploaded '{folder_path}' to '{remote_path}' in the Modal volume.")
+        except Exception as e:
+            logging.error(f"Failed to upload '{folder_path}' to '{remote_path}': {e}")
+
+if __name__ == "__main__":
+    import sys
+
+    if len(sys.argv) != 5:
+        print("Usage: python script_name.py <place> <building> <floor> <local folder path>")
+        sys.exit(1)
+
+    arg1, arg2, arg3, folder_path = sys.argv[1], sys.argv[2], sys.argv[3], sys.argv[4]
+    remote_path = os.path.join("data",arg1, arg2, arg3)
+
+    process_upload(remote_path, folder_path)

src/modal_functions/volume/modalvolumedata_setup.py

@@ -28,10 +28,10 @@
 bucket_name = 'vis4ion'
 
 files = {
-    "demo_query.png": "17MzPE9TyKiNsi6G59rqLCMMd40cIK3bU",
-    "destination.json": "1sIzFujoumSsVlZqlwwO20l96ZziORP-w",
-    "hloc.yaml": "15JYLqU9Y56keMrg9ZfxwfbkbL6_haYpx",
-    "MapConnection_Graph.pkl": "199xZSc9jSajiCqzDW_AzhuqOp_YS41fZ",
+    # "demo_query.png": "17MzPE9TyKiNsi6G59rqLCMMd40cIK3bU",
+    # "destination.json": "1sIzFujoumSsVlZqlwwO20l96ZziORP-w",
+    # "hloc.yaml": "15JYLqU9Y56keMrg9ZfxwfbkbL6_haYpx",
+    # "MapConnection_Graph.pkl": "199xZSc9jSajiCqzDW_AzhuqOp_YS41fZ",
 }
 
 @app.function(volumes={"/files": volume})   ## to create necesasry directories 
@@ -100,9 +100,41 @@ def checkAndDownload_file_from_remoteStorage():   ## download the data to the re
 
     logging.info("All files downloaded successfully from google drive and s3 bucket.")
 
+@app.function(volumes={"/files": volume})
+def rearrange_files_and_folders():
+    items_to_rearrange = [
+    # ("/files/data/6_floor", "/files/data/New_York_City/6_floor"),  # File
+    # ("/files/data/global_features.h5", "/files/data/New_York_City/global_features.h5"),  # File
+    # ("/files/data/NYISE_VC", "/files/data/New_York_City/NYISE_VC"),    # Folder
+    # ("/files/data/save.pkl", "/files/data/New_York_City/save.pkl"),
+    # ("/files/data/MapConnnection_Graph.pkl", "/files/data/New_York_City/MapConnnection_Graph.pkl"),
+]
+    for source_path, destination_path in items_to_rearrange:
+        try:
+            if os.path.isfile(source_path):
+
+                logging.info(f"Moving file from {source_path} to {destination_path}")
+                os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+                shutil.move(source_path, destination_path)
+                logging.info(f"Successfully moved file from {source_path} to {destination_path}")
+
+            elif os.path.isdir(source_path):
+
+                logging.info(f"Moving folder from {source_path} to {destination_path}")
+                os.makedirs(os.path.dirname(destination_path), exist_ok=True)
+                shutil.move(source_path, destination_path)
+                logging.info(f"Successfully moved folder from {source_path} to {destination_path}")
+
+            else:
+                logging.warning(f"Source path not found: {source_path}")
+        except Exception as e:
+            logging.error(f"Failed to move {source_path} to {destination_path}. Error: {e}")
+
+    logging.info("All files and folders have been rearranged successfully.")
 
 
 if __name__ == "__main__":
-    with app.run():
+    with app.run(detach=True):
         create_directories.remote()
         checkAndDownload_file_from_remoteStorage.remote()
+        rearrange_files_and_folders.remote()

src/modal_functions/volume/readme.md

@@ -0,0 +1,11 @@
+## upload from local to modal 
+- refer this file ``` src/modal_functions/add_new_map.py```
+- submit the parameters while executing 
+    ```
+    python add_new_map.py <place> <building> <floor> <local_folder_path>
+    ```
+- Example
+  ```
+  python add_new_map.py New_york_city nyc_hospital 17_floor_dog home/users/user1/.....( folder path that contain all the files for mentioned place, building, floor)
+   ```
+