Merge pull request #203 from seqeralabs/dev

ejseqera · web-flow · commit 870c4f7689eb · 2025-02-21T10:30:40.000-05:00
Release v0.5.3
diff --git a/README.md b/README.md
@@ -486,6 +486,7 @@ Values in the provided environment file will override any existing environment v
 We have provided template YAML files for each of the entities that can be created on Seqera Platform. These can be found in the [`templates/`](https://github.com/seqeralabs/blob/main/seqera-kit/templates) directory and should form a good starting point for you to add your own customization:
 
 - [organizations.yml](./templates/organizations.yml)
+- [members.yml](./templates/members.yml)
 - [teams.yml](./templates/teams.yml)
 - [workspaces.yml](./templates/workspaces.yml)
 - [participants.yml](./templates/participants.yml)
@@ -497,6 +498,8 @@ We have provided template YAML files for each of the entities that can be create
 - [labels.yml](./templates/labels.yml)
 - [pipelines.yml](./templates/pipelines.yml)
 - [launch.yml](./templates/launch.yml)
+- [data-links.yml](./templates/data-links.yml)
+- [studios.yml](./templates/studios.yml)
 
 ## Real world example
 
diff --git a/seqerakit/cli.py b/seqerakit/cli.py
@@ -272,6 +272,8 @@ def main(args=None):
             "secrets",
             "actions",
             "datasets",
+            "studios",
+            "data-links",
         ],
     )
 
diff --git a/seqerakit/helper.py b/seqerakit/helper.py
@@ -128,6 +128,8 @@ def parse_all_yaml(file_paths, destroy=False, targets=None):
         "datasets",
         "pipelines",
         "launch",
+        "data-links",
+        "studios",
     ]
 
     # Reverse the order of resources to delete if destroy is True
@@ -175,7 +177,11 @@ def parse_block(block_name, item):
 def parse_generic_block(item):
     cmd_args = []
     for key, value in item.items():
-        cmd_args.extend([f"--{key}", str(value)])
+        if isinstance(value, bool):
+            if value:
+                cmd_args.append(f"--{key}")
+        else:
+            cmd_args.extend([f"--{key}", str(value)])
     return cmd_args
 
 
diff --git a/seqerakit/overwrite.py b/seqerakit/overwrite.py
@@ -34,6 +34,7 @@ class Overwrite:
         "datasets",
         "actions",
         "pipelines",
+        "studios",
     ]
 
     def __init__(self, sp):
@@ -62,7 +63,7 @@ def __init__(self, sp):
             },
             "labels": {
                 "keys": ["name", "value", "workspace"],
-                "method_args": self._get_label_args,
+                "method_args": lambda args: self._get_id_resource_args("labels", args),
                 "name_key": "name",
             },
             "members": {
@@ -85,6 +86,13 @@ def __init__(self, sp):
                 "method_args": self._get_workspace_args,
                 "name_key": "workspaceName",
             },
+            "data-links": {
+                "keys": ["name", "workspace"],
+                "method_args": lambda args: self._get_id_resource_args(
+                    "data-links", args
+                ),
+                "name_key": "name",
+            },
         }
 
     def handle_overwrite(self, block, args, overwrite=False, destroy=False):
@@ -197,14 +205,17 @@ def _get_workspace_args(self, args):
         workspace_id = self._find_workspace_id(args["organization"], args["name"])
         return ("delete", "--id", str(workspace_id))
 
-    def _get_label_args(self, args):
+    def _get_id_resource_args(self, resource_type, args):
         """
         Returns a list of arguments for the delete() method for labels. The
-        label_id used to delete will be retrieved using the _find_label_id()
-        method.
+        label_id used to delete will be retrieved using the _find_id() method.
         """
-        label_id = self._find_label_id(args["name"], args["value"])
-        return ("delete", "--id", str(label_id), "-w", args["workspace"])
+        if resource_type == "labels":
+            resource_id = self._find_id(resource_type, args["name"], args["value"])
+        else:  # data-links
+            resource_id = self._find_id(resource_type, args["name"])
+
+        return ("delete", "--id", str(resource_id), "-w", args["workspace"])
 
     def _get_generic_deletion_args(self, args):
         """
@@ -252,6 +263,7 @@ def _get_json_data(self, block, args, keys_to_get):
             elif block in Overwrite.generic_deletion or block in {
                 "participants",
                 "labels",
+                "data-links",
             }:
                 sp_args = self._get_values_from_cmd_args(args, keys_to_get)
                 with self.sp.suppress_output():
@@ -326,16 +338,41 @@ def _find_workspace_id(self, organization, workspace_name):
                 return workspace_id
         return None
 
-    def _find_label_id(self, label_name, label_value):
+    def _find_id(self, resource_type, name, value=None):
         """
-        Custom method to find a label ID in a nested dictionary with a given
-        workspace name. This ID will be used to delete the label.
+        Finds the unique identifier (ID) for a Seqera Platform resource by searching
+        through the cached JSON data. This method is necessary because certain Platform
+        resources must be deleted using their ID rather than their name.
+
+        Args:
+            resource_type (str): Type of resource to search for. Currently supports:
+                - 'labels': Platform labels that require both name and value matching
+                - 'data-links': Data link resources that only require name matching
+            name (str): Name of the resource to find
+            value (str, optional): For labels only, the value field that must match
+                along with the name. Defaults to None for non-label resources.
+
+        Returns:
+            str: The unique identifier (ID) of the matching resource if found
+            None: If no matching resource is found
+
+        Note:
+            - For labels, both name and value must match to find the correct ID
+            - For data-links, only the name needs to match
+            - The method uses cached JSON data from previous API calls to avoid
+              redundant requests to the Platform
         """
         jsondata = json.loads(self.cached_jsondata)
-        labels = jsondata["labels"]
-        for label in labels:
-            if label.get("name") == utils.resolve_env_var(label_name) and label.get(
-                "value"
-            ) == utils.resolve_env_var(label_value):
-                return label.get("id")
+        json_key = "dataLinks" if resource_type == "data-links" else resource_type
+        resources = jsondata[json_key]
+
+        for resource in resources:
+            if resource_type == "labels":
+                if resource.get("name") == utils.resolve_env_var(name) and resource.get(
+                    "value"
+                ) == utils.resolve_env_var(value):
+                    return resource.get("id")
+            elif resource_type == "data-links":
+                if resource.get("name") == utils.resolve_env_var(name):
+                    return resource.get("id")
         return None
diff --git a/setup.py b/setup.py
@@ -2,7 +2,7 @@
 
 from setuptools import find_packages, setup
 
-VERSION = "0.5.2"
+VERSION = "0.5.3"
 
 with open("README.md") as f:
     readme = f.read()
diff --git a/templates/data-links.yml b/templates/data-links.yml
@@ -0,0 +1,19 @@
+## To see the full list of options available run: "tw data-links add"
+
+# Creating a private data link
+data-links:
+  - name: "my-data-link"                                              # required
+    workspace: "my_organization/my_workspace"                         # required
+    description: "Data link for my results"                           # optional
+    provider: "aws"                                                   # required
+    credentials: "my_aws_credentials"                                 # optional
+    uri: "s3://my-bucket/my-results/"                                 # required
+    overwrite: False                                                  # optional
+
+# Creating a public data link
+  - name: "1000-genomes"                                              # required
+    workspace: "my_organization/my_workspace"                         # required
+    description: "Public data link to 1000 genomes public bucket"     # optional
+    provider: "aws"                                                   # required
+    uri: "s3://1000genomes"                                           # required
+    overwrite: False                                                  # optional
diff --git a/templates/studios.yml b/templates/studios.yml
@@ -0,0 +1,48 @@
+## To see the full list of options available run: "tw studios add"
+## The options required to create studios can be specified:
+##   1. With a provided template using `template:` option
+##   2. With a custom template using `custom-template:` option
+##   3. A template image customized with conda packages using `conda-env-yml:` option
+##   4. Mounting data resources using `mount-data-resource-refs:` option
+##   5. Mounting data using data link names using `mount-data:` option
+
+# Mounting with a template image
+studios:
+  - name: 'rstudio_environment'                                               # required
+    description: 'An RStudio environment for testing'                         # optional
+    workspace: 'my_organization/my_workspace'                                 # required
+    template: 'public.cr.seqera.io/platform/data-studio-rstudio:4.4.1-u1-0.7' # required
+    compute-env: 'my_aws_compute_environment'                                 # required
+    gpu: 1                                                                    # optional
+    cpu: 2                                                                    # optional
+    memory: 4096                                                              # optional
+    autoStart: True                                                           # optional
+    mount-data-resource-refs: 's3://my_bucket/my_data'                        # optional, comma separated list
+    overwrite: False                                                          # optional
+
+# Mounting with a custom template
+  - name: 'rstudio_environment_custom_template'                                 # required
+    description: 'An RStudio environment built with a custom image for testing' # optional
+    workspace: 'my_organization/my_workspace'                                   # required
+    custom-template: 'my-registry/my-template:latest'                           # required
+    compute-env: 'my_aws_compute_environment'                                   # required
+    gpu: 1                                                                      # optional
+    cpu: 2                                                                      # optional
+    memory: 4096                                                                # optional
+    autoStart: True                                                             # optional
+    mount-data-resource-refs: 's3://my_bucket/my_data'                          # optional, comma separated list
+    overwrite: False                                                            # optional
+
+# Mounting with a template image customized with conda packages
+  - name: 'rstudio_environment_conda_packages'                                  # required
+    description: 'An RStudio environment built with conda packages for testing' # optional
+    workspace: 'my_organization/my_workspace'                                   # required
+    template: 'public.cr.seqera.io/platform/data-studio-rstudio:4.4.1-u1-0.7'   # required
+    conda-env-yml: './templates/rstudio_environment.yml'                        # required
+    compute-env: 'my_aws_compute_environment'                                   # required
+    gpu: 1                                                                      # optional
+    cpu: 2                                                                      # optional
+    memory: 4096                                                                # optional
+    autoStart: True                                                             # optional
+    mount-data-resource-refs: 's3://my_bucket/my_data'                          # optional, comma separated list
+    overwrite: False                                                            # optional
diff --git a/tests/unit/test_helper.py b/tests/unit/test_helper.py
@@ -333,6 +333,88 @@ def test_create_mock_members_yaml(mock_yaml_file):
     assert result["members"] == expected_block_output
 
 
+def test_create_mock_studios_yaml(mock_yaml_file):
+    test_data = {
+        "studios": [
+            {
+                "name": "test_studio1",
+                "workspace": "my_organization/my_workspace",
+                "template": "public.seqera.io/public/data-studio-rstudio:4.4.1",
+                "compute-env": "my_computeenv",
+                "cpu": 2,
+                "memory": 4096,
+                "autoStart": False,
+                "overwrite": True,
+                "mount-data-ids": "v1-user-bf73f9d33997f93a20ee3e6911779951",
+            }
+        ]
+    }
+
+    expected_block_output = [
+        {
+            "cmd_args": [
+                "--compute-env",
+                "my_computeenv",
+                "--cpu",
+                "2",
+                "--memory",
+                "4096",
+                "--mount-data-ids",
+                "v1-user-bf73f9d33997f93a20ee3e6911779951",
+                "--name",
+                "test_studio1",
+                "--template",
+                "public.seqera.io/public/data-studio-rstudio:4.4.1",
+                "--workspace",
+                "my_organization/my_workspace",
+            ],
+            "overwrite": True,
+        }
+    ]
+
+    file_path = mock_yaml_file(test_data)
+    result = helper.parse_all_yaml([file_path])
+    print(f"debug - result: {result}")
+    assert "studios" in result
+    assert result["studios"] == expected_block_output
+
+
+def test_create_mock_data_links_yaml(mock_yaml_file):
+    test_data = {
+        "data-links": [
+            {
+                "name": "test_data_link1",
+                "workspace": "my_organization/my_workspace",
+                "provider": "aws",
+                "credentials": "my_credentials",
+                "uri": "s3://scidev-playground-eu-west-2/esha/nf-core-scrnaseq/",
+                "overwrite": True,
+            }
+        ]
+    }
+    expected_block_output = [
+        {
+            "cmd_args": [
+                "--credentials",
+                "my_credentials",
+                "--name",
+                "test_data_link1",
+                "--provider",
+                "aws",
+                "--uri",
+                "s3://scidev-playground-eu-west-2/esha/nf-core-scrnaseq/",
+                "--workspace",
+                "my_organization/my_workspace",
+            ],
+            "overwrite": True,
+        }
+    ]
+    file_path = mock_yaml_file(test_data)
+    result = helper.parse_all_yaml([file_path])
+    assert "data-links" in result
+    assert result["data-links"] == expected_block_output
+
+
 def test_empty_yaml_file(mock_yaml_file):
     test_data = {}
     file_path = mock_yaml_file(test_data)
diff --git a/tests/unit/test_overwrite.py b/tests/unit/test_overwrite.py
@@ -142,7 +142,6 @@ def test_participant_deletion(self):
 
     @patch("seqerakit.utils.resolve_env_var")
     def test_organization_deletion_with_env_var(self, mock_resolve_env_var):
-
         args = ["--name", "${ORG_NAME}"]
 
         # Setup environment variable mock

Original file line number	Diff line number	Diff line change
`@@ -272,6 +272,8 @@ def main(args=None):`
`272`	`272`	`"secrets",`
`273`	`273`	`"actions",`
`274`	`274`	`"datasets",`
	`275`	`+ "studios",`
	`276`	`+ "data-links",`
`275`	`277`	`],`
`276`	`278`	`)`
`277`	`279`