feat(schematic): refactor storage endpoints (#2274)

* refactored endpoints

* refactored endpoints

* change asset type decription

.devcontainer/devcontainer.json

@@ -51,6 +51,7 @@
         "mtxr.sqltools-driver-mysql",
         "mtxr.sqltools-driver-pg",
         "mtxr.sqltools",
+        "njpwerner.autodocstring",
         "nrwl.angular-console",
         "pranaygp.vscode-css-peek",
         "ritwickdey.LiveServer",

apps/schematic/api/schematic_api/controllers/storage_controller.py

@@ -32,18 +32,18 @@ def get_asset_view_json(asset_view_id, asset_type):  # noqa: E501
 
 
 def get_dataset_files(
-    dataset_id, asset_view_id, asset_type, file_names=None, use_full_file_path=None
+    dataset_id, asset_type, asset_view_id, file_names=None, use_full_file_path=None
 ):  # noqa: E501
     """Gets all files associated with a dataset.
 
     Gets all files associated with a dataset. # noqa: E501
 
     :param dataset_id: The ID of a dataset.
     :type dataset_id: str
-    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
-    :type asset_view_id: str
     :param asset_type: Type of asset, such as Synapse
     :type asset_type: dict | bytes
+    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
+    :type asset_view_id: str
     :param file_names: A list of file names used to filter the output.
     :type file_names: List[str]
     :param use_full_file_path: Whether or not to return the full path of output, or just the basename.
@@ -54,28 +54,28 @@ def get_dataset_files(
     if connexion.request.is_json:
         asset_type = AssetType.from_dict(connexion.request.get_json())  # noqa: E501
     return storage_controller_impl.get_dataset_files(
-        dataset_id, asset_view_id, asset_type, file_names, use_full_file_path
+        dataset_id, asset_type, asset_view_id, file_names, use_full_file_path
     )
 
 
-def get_dataset_manifest_json(asset_type, asset_view_id, dataset_id):  # noqa: E501
+def get_dataset_manifest_json(asset_type, dataset_id, asset_view_id):  # noqa: E501
     """Gets the manifest in json form
 
     Gets the manifest in json form # noqa: E501
 
     :param asset_type: Type of asset, such as Synapse
     :type asset_type: dict | bytes
-    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
-    :type asset_view_id: str
     :param dataset_id: The ID of a dataset.
     :type dataset_id: str
+    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
+    :type asset_view_id: str
 
     :rtype: Union[object, Tuple[object, int], Tuple[object, int, Dict[str, str]]
     """
     if connexion.request.is_json:
         asset_type = AssetType.from_dict(connexion.request.get_json())  # noqa: E501
     return storage_controller_impl.get_dataset_manifest_json(
-        asset_type, asset_view_id, dataset_id
+        asset_type, dataset_id, asset_view_id
     )
 
 
@@ -96,60 +96,60 @@ def get_manifest_json(asset_type, manifest_id):  # noqa: E501
     return storage_controller_impl.get_manifest_json(asset_type, manifest_id)
 
 
-def list_projects(asset_view_id, asset_type):  # noqa: E501
-    """Gets all storage projects the current user has access to.
+def get_project_datasets(project_id, asset_type, asset_view_id):  # noqa: E501
+    """Gets all datasets in folder under a given storage project that the current user has access to.
 
-    Gets all storage projects the current user has access to. # noqa: E501
+    Gets all datasets in folder under a given storage project that the current user has access to. # noqa: E501
 
-    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
-    :type asset_view_id: str
+    :param project_id: The Synapse ID of a storage project.
+    :type project_id: str
     :param asset_type: Type of asset, such as Synapse
     :type asset_type: dict | bytes
+    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
+    :type asset_view_id: str
 
-    :rtype: Union[ProjectsPage, Tuple[ProjectsPage, int], Tuple[ProjectsPage, int, Dict[str, str]]
+    :rtype: Union[DatasetsPage, Tuple[DatasetsPage, int], Tuple[DatasetsPage, int, Dict[str, str]]
     """
     if connexion.request.is_json:
         asset_type = AssetType.from_dict(connexion.request.get_json())  # noqa: E501
-    return storage_controller_impl.list_projects(asset_view_id, asset_type)
+    return storage_controller_impl.get_project_datasets(
+        project_id, asset_type, asset_view_id
+    )
 
 
-def list_storage_project_datasets(project_id, asset_view_id, asset_type):  # noqa: E501
-    """Gets all datasets in folder under a given storage project that the current user has access to.
+def get_project_manifests(project_id, asset_type, asset_view_id):  # noqa: E501
+    """Gets all manifests in a project folder that users have access to
 
-    Gets all datasets in folder under a given storage project that the current user has access to. # noqa: E501
+    Gets all manifests in a project folder that the current user has access to. # noqa: E501
 
     :param project_id: The Synapse ID of a storage project.
     :type project_id: str
-    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
-    :type asset_view_id: str
     :param asset_type: Type of asset, such as Synapse
     :type asset_type: dict | bytes
+    :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
+    :type asset_view_id: str
 
-    :rtype: Union[DatasetsPage, Tuple[DatasetsPage, int], Tuple[DatasetsPage, int, Dict[str, str]]
+    :rtype: Union[ManifestsPage, Tuple[ManifestsPage, int], Tuple[ManifestsPage, int, Dict[str, str]]
     """
     if connexion.request.is_json:
         asset_type = AssetType.from_dict(connexion.request.get_json())  # noqa: E501
-    return storage_controller_impl.list_storage_project_datasets(
-        project_id, asset_view_id, asset_type
+    return storage_controller_impl.get_project_manifests(
+        project_id, asset_type, asset_view_id
     )
 
 
-def list_storage_project_manifests(project_id, asset_view_id, asset_type):  # noqa: E501
-    """Gets all manifests in a project folder that users have access to
+def get_projects(asset_view_id, asset_type):  # noqa: E501
+    """Gets all storage projects the current user has access to.
 
-    Gets all manifests in a project folder that the current user has access to. # noqa: E501
+    Gets all storage projects the current user has access to. # noqa: E501
 
-    :param project_id: The Synapse ID of a storage project.
-    :type project_id: str
     :param asset_view_id: ID of view listing all project data assets. E.g. for Synapse this would be the Synapse ID of the fileview listing all data assets for a given project
     :type asset_view_id: str
     :param asset_type: Type of asset, such as Synapse
     :type asset_type: dict | bytes
 
-    :rtype: Union[ManifestsPage, Tuple[ManifestsPage, int], Tuple[ManifestsPage, int, Dict[str, str]]
+    :rtype: Union[ProjectsPage, Tuple[ProjectsPage, int], Tuple[ProjectsPage, int, Dict[str, str]]
     """
     if connexion.request.is_json:
         asset_type = AssetType.from_dict(connexion.request.get_json())  # noqa: E501
-    return storage_controller_impl.list_storage_project_manifests(
-        project_id, asset_view_id, asset_type
-    )
+    return storage_controller_impl.get_projects(asset_view_id, asset_type)

apps/schematic/api/schematic_api/controllers/storage_controller_impl.py

@@ -106,8 +106,8 @@ def get_dataset_files_from_schematic(
 @handle_exceptions
 def get_dataset_files(
     dataset_id: str,
-    asset_view_id: str,
     asset_type: str,
+    asset_view_id: str,
     file_names: Optional[list[str]] = None,
     use_full_file_path: bool = False,
 ) -> tuple[Union[FilesPage, BasicError], int]:
@@ -186,7 +186,9 @@ def get_dataset_manifest_from_schematic(
 
 @handle_exceptions
 def get_dataset_manifest_json(
-    asset_type: str, asset_view_id: str, dataset_id: str
+    asset_type: str,
+    dataset_id: str,
+    asset_view_id: str,
 ) -> tuple[Union[str, BasicError], int]:
     """Gets a manifest in json form
 
@@ -253,57 +255,9 @@ def get_manifest_json(
     return result, status
 
 
-def get_projects(asset_type: str) -> list[tuple[str, str]]:
-    """Gets a list of projects
-
-    Args:
-        asset_type (str): The type of asset, ie "synapse"
-
-    Returns:
-        list[tuple(str, str)]: A list of projects in tuple form
-    """
-    access_token = get_access_token()
-    asset_type_object = get_asset_storage_class(asset_type)
-    store = asset_type_object(access_token=access_token)
-    return store.getStorageProjects()
-
-
-@handle_exceptions
-def list_projects(
-    asset_view_id: str, asset_type: str
-) -> tuple[Union[ProjectsPage, BasicError], int]:
-    """Attempts to get a list of projects the user has access to
-
-    Args:
-        asset_view_id (str): The id for the asset view of the project
-        asset_type (str): The type of asset, ie "synapse"
-
-    Returns:
-        tuple[Union[ProjectsPage, BasicError], int]: A tuple
-          The first item is either the projects or an error object
-          The second item is the response status
-    """
-
-    CONFIG.synapse_master_fileview_id = asset_view_id
-    project_tuples = get_projects(asset_type)
-    projects = [Project(id=item[0], name=item[1]) for item in project_tuples]
-
-    page = ProjectsPage(
-        number=0,
-        size=100,
-        total_elements=len(projects),
-        total_pages=1,
-        has_next=False,
-        has_previous=False,
-        projects=projects,
-    )
-    result: Union[ProjectsPage, BasicError] = page
-    status = 200
-
-    return result, status
-
-
-def get_project_datasets(project_id: str, asset_type: str) -> list[tuple[str, str]]:
+def get_project_datasets_from_schematic(
+    project_id: str, asset_type: str
+) -> list[tuple[str, str]]:
     """Gets a list of datasets from the project
 
     Args:
@@ -320,8 +274,8 @@ def get_project_datasets(project_id: str, asset_type: str) -> list[tuple[str, st
 
 
 @handle_exceptions
-def list_storage_project_datasets(
-    project_id: str, asset_view_id: str, asset_type: str
+def get_project_datasets(
+    project_id: str, asset_type: str, asset_view_id: str
 ) -> tuple[Union[DatasetsPage, BasicError], int]:
     """Attempts to get a list of datasets from a Synapse project
 
@@ -337,7 +291,7 @@ def list_storage_project_datasets(
     """
 
     CONFIG.synapse_master_fileview_id = asset_view_id
-    dataset_tuples = get_project_datasets(project_id, asset_type)
+    dataset_tuples = get_project_datasets_from_schematic(project_id, asset_type)
     datasets = [Dataset(id=item[0], name=item[1]) for item in dataset_tuples]
 
     page = DatasetsPage(
@@ -355,7 +309,7 @@ def list_storage_project_datasets(
     return result, status
 
 
-def get_project_manifests(
+def get_project_manifests_from_schematic(
     project_id: str, asset_type: str
 ) -> list[tuple[tuple[str, str], tuple[str, str], tuple[str, str]]]:
     """Gets a list of manifests from the project
@@ -374,8 +328,8 @@ def get_project_manifests(
 
 
 @handle_exceptions
-def list_storage_project_manifests(
-    project_id: str, asset_view_id: str, asset_type: str
+def get_project_manifests(
+    project_id: str, asset_type: str, asset_view_id: str
 ) -> tuple[Union[ManifestsPage, BasicError], int]:
     """Attempts to get a list of manifests from a Synapse project
 
@@ -391,7 +345,7 @@ def list_storage_project_manifests(
     """
     # load config
     CONFIG.synapse_master_fileview_id = asset_view_id
-    project_manifests = get_project_manifests(project_id, asset_type)
+    project_manifests = get_project_manifests_from_schematic(project_id, asset_type)
     manifests = [
         Manifest(
             name=item[1][1],
@@ -416,3 +370,53 @@ def list_storage_project_manifests(
     status = 200
 
     return result, status
+
+
+def get_projects_from_schematic(asset_type: str) -> list[tuple[str, str]]:
+    """Gets a list of projects
+
+    Args:
+        asset_type (str): The type of asset, ie "synapse"
+
+    Returns:
+        list[tuple(str, str)]: A list of projects in tuple form
+    """
+    access_token = get_access_token()
+    asset_type_object = get_asset_storage_class(asset_type)
+    store = asset_type_object(access_token=access_token)
+    return store.getStorageProjects()
+
+
+@handle_exceptions
+def get_projects(
+    asset_view_id: str, asset_type: str
+) -> tuple[Union[ProjectsPage, BasicError], int]:
+    """Attempts to get a list of projects the user has access to
+
+    Args:
+        asset_view_id (str): The id for the asset view of the project
+        asset_type (str): The type of asset, ie "synapse"
+
+    Returns:
+        tuple[Union[ProjectsPage, BasicError], int]: A tuple
+          The first item is either the projects or an error object
+          The second item is the response status
+    """
+
+    CONFIG.synapse_master_fileview_id = asset_view_id
+    project_tuples = get_projects_from_schematic(asset_type)
+    projects = [Project(id=item[0], name=item[1]) for item in project_tuples]
+
+    page = ProjectsPage(
+        number=0,
+        size=100,
+        total_elements=len(projects),
+        total_pages=1,
+        has_next=False,
+        has_previous=False,
+        projects=projects,
+    )
+    result: Union[ProjectsPage, BasicError] = page
+    status = 200
+
+    return result, status