Client script for uploading tasks

README.rst

@@ -33,7 +33,7 @@ To use Girder Worker:
 
 .. code-block:: bash
 
-  pip install girder-slicer-cli-web[worker]
+  pip install 'girder-slicer-cli-web[worker]'
   GW_DIRECT_PATHS=true girder_worker -l info -Ofair --prefetch-multiplier=1
 
 The first time you start Girder, you'll need to configure it with at least one user and one assetstore (see the Girder_ documentation).  Additionally, it is recommended that you install some dockerized tasks, such as the HistomicsTK_ algorithms.  This can be done going to the Admin Console, Plugins, Slicer CLI Web settings.  Set a default task upload folder, then import the `dsarchive/histomicstk:latest` docker image.
@@ -44,9 +44,24 @@ Girder Plugin
 Importing Docker Images
 =======================
 
-When installed in Girder, an admin user can go to the Admin Console -> Plugins -> Slicer CLI Web to add Docker images.  Select a Docker image and an existing folder and then select Import Image.  Slicer CLI Web will pull the Docker image if it is not available on the Girder machine.
+Once a docker image has been created and pushed to Docker Hub, you can register the image's CLI as a set of tasks on the server. To do so,
+use the client upload script bundled with this tool. To install it, run:
+
+.. code-block:: bash
+
+  pip install 'girder-slicer-cli-web[client]'
+
+Create an API key with the "Manage Slicer CLI tasks" scope, and set it in your environment and run a command like this example:
+
+.. code-block:: bash
+
+  GIRDER_API_KEY=my_key_vale upload-slicer-cli-task https://my-girder-host.com/api/v1 641b8578cdcf8f129805524b my-slicer-cli-image:latest
+
+The first argument of this command is the API URL of the server, the second is a Girder folder ID where the tasks will live, and the
+last argument is the docker image identifier. (If the image does not exist locally it will be pulled.) If you just want to create a
+single CLI task rather than all tasks from ``--list_cli``, you can pass ``--cli=CliName``. If you wish to replace the existing tasks
+with the latest specifications, also pass the ``--replace`` flag to the command.
 
-For each docker image that is imported, a folder is created with the image tag.  Within this folder, a subfolder is created with the image version.  The subfolder will have one item per CLI that the Docker image reports.  These items can be moved after they have been imported, just like standard Girder items.
 
 Running CLIs
 ============
@@ -195,4 +210,3 @@ If the local (server) environment has any environment variables that begin with
 .. _Girder: http://girder.readthedocs.io/en/latest/
 .. _Girder Worker: https://girder-worker.readthedocs.io/en/latest/
 .. _HistomicsTK: https://github.com/DigitalSlideArchive/HistomicsTK
-

setup.py

@@ -66,14 +66,17 @@ def prerelease_local_scheme(version):
     ],
     extras_require={
         'girder': [
-            'docker>=2.6.0',
             'girder>=3.0.4',
             'girder-jobs>=3.0.3',
             'girder-worker[girder]>=0.6.0',
         ],
         'worker': [
             'docker>=2.6.0',
             'girder-worker[worker]>=0.6.0',
+        ],
+        'client': [
+            'click',
+            'girder-client',
         ]
     },
     entry_points={
@@ -82,6 +85,9 @@ def prerelease_local_scheme(version):
         ],
         'girder_worker_plugins': [
             'slicer_cli_web = slicer_cli_web.girder_worker_plugin:SlicerCLIWebWorkerPlugin'
+        ],
+        'console_scripts': [
+            'upload-slicer-cli-task = slicer_cli_web.upload_slicer_cli_task:upload_slicer_cli_task'
         ]
     },
     python_requires='>=3.6',

slicer_cli_web/__init__.py

@@ -37,5 +37,6 @@
     # package is not installed
     pass
 
-
 __license__ = 'Apache 2.0'
+
+TOKEN_SCOPE_MANAGE_TASKS = 'slicer_cli_web.manage_tasks'

slicer_cli_web/docker_resource.py

@@ -15,25 +15,28 @@
 #############################################################################
 
 
+from base64 import b64decode
 import json
 import os
 import re
 
 from girder.api import access
 from girder.api.describe import Description, autoDescribeRoute, describeRoute
-from girder.api.rest import setRawResponse, setResponseHeader
+from girder.api.rest import setRawResponse, setResponseHeader, filtermodel
 from girder.api.v1.resource import Resource, RestException
 from girder.constants import AccessType, SortDir
 from girder.exceptions import AccessException
+from girder.models.folder import Folder
 from girder.models.item import Item
 from girder.utility import path as path_util
 from girder.utility.model_importer import ModelImporter
 from girder.utility.progress import setResponseTimeLimit
 from girder_jobs.constants import JobStatus
 from girder_jobs.models.job import Job
 
+from . import TOKEN_SCOPE_MANAGE_TASKS
 from .config import PluginSettings
-from .models import CLIItem, DockerImageItem, DockerImageNotFoundError
+from .models import CLIItem, DockerImageItem, DockerImageNotFoundError, parser
 from .rest_slicer_cli import genRESTEndPointsForSlicerCLIsForItem
 
 
@@ -51,6 +54,7 @@ def __init__(self, name):
         self.resourceName = name
         self.jobType = 'slicer_cli_web_job'
         self.route('PUT', ('docker_image',), self.setImages)
+        self.route('POST', ('cli',), self.createOrReplaceCli)
         self.route('DELETE', ('docker_image',), self.deleteImage)
         self.route('GET', ('docker_image',), self.getDockerImages)
 
@@ -62,6 +66,7 @@ def __init__(self, name):
 
         self.route('GET', ('path_match', ), self.getMatchingResource)
 
+
         self._generateAllItemEndPoints()
 
     @access.public
@@ -183,7 +188,7 @@ def parseImageNameList(self, param):
                 raise RestException('Image %s does not have a tag or digest' % img)
         return nameList
 
-    @access.admin
+    @access.admin(scope=TOKEN_SCOPE_MANAGE_TASKS)
     @describeRoute(
         Description('Add one or a list of images')
         .notes('Must be a system administrator to call this.')
@@ -210,6 +215,37 @@ def setImages(self, params):
             raise RestException('no upload folder given or defined by default')
         return self._createPutImageJob(nameList, folder, params.get('pull', None))
 
+    @access.admin(scope=TOKEN_SCOPE_MANAGE_TASKS)
+    @filtermodel(Item)
+    @autoDescribeRoute(
+        Description('Add or replace an item task.')
+        .notes('Must be a system administrator to call this.')
+        .modelParam('folder', 'The folder ID to upload the task to.', paramType='formData',
+                    model=Folder, level=AccessType.WRITE)
+        .param('image', 'The docker image identifier.')
+        .param('name', 'The name of the item to create or replace.')
+        .param('replace', 'Whether to replace an existing item with this name.', dataType='boolean')
+        .param('spec', 'Base64-encoded XML spec of the CLI.')
+        .errorResponse('You are not a system administrator.', 403)
+    )
+    def createOrReplaceCli(self, folder: dict, image: str, name: str, replace: bool, spec: str):
+        try:
+            spec = b64decode(spec).decode()
+        except ValueError:
+            raise RestException('The CLI spec must be base64-encoded UTF-8.')
+
+        item = Item().createItem(
+            name, creator=self.getCurrentUser(), folder=folder, reuseExisting=replace
+        )
+        metadata = dict(
+            slicerCLIType='task',
+            type='Unknown',  # TODO does "type" matter behaviorally? If so get it from the client
+            digest=None,  # TODO should we support this?
+            image=image,
+            **parser._parse_xml_desc(item, self.getCurrentUser(), spec)
+        )
+        return Item().setMetadata(item, metadata)
+
     def _createPutImageJob(self, nameList, baseFolder, pull=False):
         job = Job().createLocalJob(
             module='slicer_cli_web.image_job',

slicer_cli_web/girder_plugin.py

@@ -18,12 +18,12 @@
 import json
 
 from girder import events, logger
-from girder.constants import AccessType
+from girder.constants import AccessType, TokenScope
 from girder.plugin import GirderPlugin, getPlugin
 from girder_jobs.constants import JobStatus
 from girder_jobs.models.job import Job
 
-from . import worker_tools
+from . import TOKEN_SCOPE_MANAGE_TASKS
 from .docker_resource import DockerResource
 from .models import DockerImageItem
 
@@ -55,6 +55,10 @@ def load(self, info):
         except Exception:
             logger.info('Girder working is unavailable')
 
+        TokenScope.describeScope(
+            TOKEN_SCOPE_MANAGE_TASKS, name='Manage Slicer CLI tasks',
+            description='Create / edit Slicer CLI docker tasks', admin=True)
+
         DockerImageItem.prepare()
 
         # resource name must match the attribute added to info[apiroot]
@@ -79,4 +83,3 @@ def load(self, info):
                 pass
         if count:
             logger.info('Marking %d old job(s) as cancelled' % count)
-        worker_tools.start()

slicer_cli_web/upload_slicer_cli_task.py

@@ -0,0 +1,49 @@
+import base64
+import json
+import os
+import subprocess
+from typing import Optional
+
+import click
+from girder_client import GirderClient
+
+
+def upload_cli(gc: GirderClient, image_name: str, replace: bool, cli_name: str, folder_id: str):
+    output = subprocess.check_output(['docker', 'run', image_name, cli_name, '--xml'])
+    gc.post(f'slicer_cli_web/cli', data={
+        'folder': folder_id,
+        'image': image_name,
+        'name': cli_name,
+        'replace': str(replace),
+        'spec': base64.b64encode(output),
+    })
+
+
+@click.command()
+@click.argument('api_url')
+@click.argument('folder_id')
+@click.argument('image_name')
+@click.option('--cli', help='Push a single CLI with the given name', default=None)
+@click.option('--replace', is_flag=True, help='Replace existing item if it exists', default=False)
+def upload_slicer_cli_task(api_url: str, folder_id: str, image_name: str, cli: Optional[str], replace: bool):
+    if 'GIRDER_API_KEY' not in os.environ:
+        raise Exception('Please set GIRDER_API_KEY in your environment.')
+
+    gc = GirderClient(apiUrl=api_url)
+    gc.authenticate(apiKey=os.environ['GIRDER_API_KEY'])
+
+    output = subprocess.check_output(['docker', 'run', image_name, '--list_cli'])
+    cli_list_json: dict = json.loads(output)
+
+    # The keys are the names of each CLI in the image
+    if cli:  # upload one
+        if cli not in cli_list_json:
+            raise ValueError('Invalid CLI name, not found in image CLI list.')
+        upload_cli(gc, image_name, replace, cli, folder_id)
+    else:  # upload all
+        for cli_name in cli_list_json:
+            upload_cli(gc, image_name, replace, cli_name, folder_id)
+
+
+if __name__ == '__main__':
+    upload_slicer_cli_task()