v0.16.22 (#122)

api.py:
* enabled namespace, name, and snapshot ID filtering for method repository listings
* new command whoami returns FireCloud user id

fiss.py: 
* updated listing calls (method_list, config_list) to enable new filters
* calls to listings updated to use filters when appropriate
* new command set_config_acl for setting ACLs on configs in the methods repo
* added capability to set method and config ACLs for groups
* meth_list and meth_exists commands updated to use argument method instead of name to be consistent with the rest of the CLI.
*  config_list updated to work on methods repo when user has a default workspace defined in their ~/.fissconfig

setup.py:
* google-auth updated to use current release

__init__.py:
* Added warning filter to silence application default warning

Makefile:
* publish updated to use twine

Makefile

@@ -78,14 +78,15 @@ reinstall:
 uninstall:
 	$(PIP) uninstall -y firecloud
 
-publish:
-	$(PYTHON) setup.py sdist upload && \
+publish: clean
+	$(PYTHON) setup.py sdist && \
+	twine upload dist/* && \
 	rm -rf build dist *.egg-info
 
 image:
 	docker build -t broadgdac/fiss .
 
 clean:
-	rm -rf build dist *.egg-info *~ */*~ *.pyc */*.pyc
+	rm -rf build dist .eggs *.egg-info *~ */*~ *.pyc */*.pyc
 
 .PHONY: help test test_cli test_one install release publish clean lintify

changelog.txt

@@ -3,6 +3,13 @@ Change Log for FISSFC: the (Fi)recloud (S)ervice (S)elector
 =======================================================================
 Terms used below:  HL = high level interface, LL = low level interface
 
+v0.16.22 - LL: enabled namespace, name, and snapshot ID filtering for
+           repository listings, new command whoami returns user id; HL: updated
+           listing calls to enable new filters, added capability to set method
+           ACLs for groups, new command set_config_acl for setting ACLs on
+           configs in the methods repo; google-auth updated to use current
+           release.
+
 v0.16.21 - api.py will attempt to update credentials on a refresh error; typo
            fixes to several HL command descriptions; setup.py now requires a
            minimum version of setuptools (40.3.0); added Groups API to LL.

firecloud/__about__.py

@@ -1,2 +1,2 @@
 # Package version
-__version__ = "0.16.21"
+__version__ = "0.16.22"

firecloud/__init__.py

@@ -1,4 +1,7 @@
 import os
+import warnings
+warnings.filterwarnings('ignore', 'Your application has authenticated using end user credentials from Google Cloud SDK. We recommend that most server applications use service accounts instead. If your application continues to use end user credentials from Cloud SDK, you might receive a "quota exceeded" or "API not enabled" error. For more information about service accounts, see https://cloud.google.com/docs/authentication/')
+
 
 # Based on https://stackoverflow.com/a/379535
 def which(program):

firecloud/api.py

@@ -20,7 +20,8 @@
 
 import google.auth
 from google.auth.exceptions import DefaultCredentialsError, RefreshError
-from google.auth.transport.requests import AuthorizedSession
+from google.auth.transport.requests import AuthorizedSession, Request
+from google.oauth2 import id_token
 
 from firecloud.errors import FireCloudServerError
 from firecloud.fccore import __fcconfig as fcconfig
@@ -31,27 +32,26 @@
 
 # Set Global Authorized Session
 __SESSION = None
+__USER_ID = None
 
 # Suppress warnings about project ID
 logging.getLogger('google.auth').setLevel(logging.ERROR)
 
 #################################################
 # Utilities
 #################################################
-def _fiss_agent_header(headers=None):
-    """ Return request headers for fiss.
-        Inserts FISS as the User-Agent.
-        Initializes __SESSION if it hasn't been set.
-
-    Args:
-        headers (dict): Include additional headers as key-value pairs
-
-    """
+def _set_session():
+    """ Sets global __SESSION and __USER_ID if they haven't been set """
     global __SESSION
+    global __USER_ID
+
     if __SESSION is None:
         try:
             __SESSION = AuthorizedSession(google.auth.default(['https://www.googleapis.com/auth/userinfo.profile',
                                                                'https://www.googleapis.com/auth/userinfo.email'])[0])
+            health()
+            __USER_ID = id_token.verify_oauth2_token(__SESSION.credentials.id_token,
+                                                     Request(session=__SESSION))['email']
         except (DefaultCredentialsError, RefreshError) as gae:
             if os.getenv('SERVER_SOFTWARE', '').startswith('Google App Engine/'):
                 raise
@@ -70,6 +70,17 @@ def _fiss_agent_header(headers=None):
                                       cpe.returncode)
                 raise gae
 
+def _fiss_agent_header(headers=None):
+    """ Return request headers for fiss.
+        Inserts FISS as the User-Agent.
+        Initializes __SESSION if it hasn't been set.
+
+    Args:
+        headers (dict): Include additional headers as key-value pairs
+
+    """
+    _set_session()
+
     fiss_headers = {"User-Agent" : FISS_USER_AGENT}
     if headers is not None:
         fiss_headers.update(headers)
@@ -130,6 +141,11 @@ def _check_response_code(response, codes):
     if response.status_code not in codes:
         raise FireCloudServerError(response.status_code, response.content)
 
+def whoami():
+    """ Return __USER_ID """
+    _set_session()
+    return __USER_ID
+
 ##############################################################
 # 1. Orchestration API calls, see https://api.firecloud.org/
 ##############################################################
@@ -465,7 +481,7 @@ def update_entity(namespace, workspace, etype, ename, updates):
 ### 1.2 Method Configurations
 ###############################
 
-def list_workspace_configs(namespace, workspace):
+def list_workspace_configs(namespace, workspace, allRepos=False):
     """List method configurations in workspace.
 
     Args:
@@ -477,7 +493,7 @@ def list_workspace_configs(namespace, workspace):
         DUPLICATE: https://api.firecloud.org/#!/Workspaces/listWorkspaceMethodConfigs
     """
     uri = "workspaces/{0}/{1}/methodconfigs".format(namespace, workspace)
-    return __get(uri)
+    return __get(uri, params={'allRepos': allRepos})
 
 def create_workspace_config(namespace, workspace, body):
     """Create method configuration in workspace.
@@ -678,32 +694,41 @@ def copy_config_to_repo(namespace, workspace, from_cnamespace,
 ### 1.3 Method Repository
 ###########################
 
-def list_repository_methods(name=None):
-    """List methods in the methods repository.
+def list_repository_methods(namespace=None, name=None, snapshotId=None):
+    """List method(s) in the methods repository.
+
+    Args:
+        namespace (str): Method Repository namespace
+        name (str): method name
+        snapshotId (int): method snapshot ID
 
     Swagger:
         https://api.firecloud.org/#!/Method_Repository/listMethodRepositoryMethods
     """
-    params = dict()
-    if name:
-        params['name'] = name
+    params = {k:v for (k,v) in locals().items() if v is not None}
     return __get("methods", params=params)
 
-def list_repository_configs():
+def list_repository_configs(namespace=None, name=None, snapshotId=None):
     """List configurations in the methods repository.
 
+    Args:
+        namespace (str): Method Repository namespace
+        name (str): config name
+        snapshotId (int): config snapshot ID
+
     Swagger:
         https://api.firecloud.org/#!/Method_Repository/listMethodRepositoryConfigurations
     """
-    return __get("configurations")
+    params = {k:v for (k,v) in locals().items() if v is not None}
+    return __get("configurations", params=params)
 
 def get_config_template(namespace, method, version):
     """Get the configuration template for a method.
 
     The method should exist in the methods repository.
 
     Args:
-        namespace (str): Methods namespace
+        namespace (str): Method's namespace
         method (str): method name
         version (int): snapshot_id of the method
 

firecloud/fiss.py

@@ -438,32 +438,72 @@ def meth_acl(args):
 @fiss_cmd
 def meth_set_acl(args):
     """ Assign an ACL role to a list of users for a workflow. """
-    acl_updates = [{"user": user, "role": args.role} for user in args.users]
+    acl_updates = [{"user": user, "role": args.role} \
+                   for user in set(expand_fc_groups(args.users)) \
+                   if user != fapi.whoami()]
 
     id = args.snapshot_id
     if not id:
         # get the latest snapshot_id for this method from the methods repo
-        r = fapi.list_repository_methods()
+        r = fapi.list_repository_methods(namespace=args.namespace,
+                                         name=args.method)
         fapi._check_response_code(r, 200)
-        versions = [m for m in r.json()
-                         if m['name'] == args.method and m['namespace'] == args.namespace]
+        versions = r.json()
         if len(versions) == 0:
             if fcconfig.verbosity:
-                eprint("method {0}/{1} not found".format(args.namespace, args.method))
+                eprint("method {0}/{1} not found".format(args.namespace,
+                                                         args.method))
             return 1
         latest = sorted(versions, key=lambda m: m['snapshotId'])[-1]
         id = latest['snapshotId']
 
-    r = fapi.update_repository_method_acl(args.namespace, args.method, id, acl_updates)
+    r = fapi.update_repository_method_acl(args.namespace, args.method, id,
+                                          acl_updates)
     fapi._check_response_code(r, 200)
     if fcconfig.verbosity:
-        print("Updated ACL for {0}/{1}:{2}".format(args.namespace, args.method, id))
+        print("Updated ACL for {0}/{1}:{2}".format(args.namespace, args.method,
+                                                   id))
     return 0
 
+def expand_fc_groups(users):
+    """ If user is a firecloud group, return all members of the group.
+    Caveat is that only group admins may do this.
+    """
+    groups = None
+    for user in users:
+        fcgroup = None
+        if '@' not in user:
+            fcgroup = user
+        elif user.lower().endswith('@firecloud.org'):
+            if groups is None:
+                r = fapi.get_groups()
+                fapi._check_response_code(r, 200)
+                groups = {group['groupEmail'].lower():group['groupName'] \
+                          for group in r.json() if group['role'] == 'Admin'}
+            if user.lower() not in groups:
+                if fcconfig.verbosity:
+                    eprint("You do not have access to the members of {}".format(user))
+                yield user
+                continue
+            else:
+                fcgroup = groups[user.lower()]
+        else:
+            yield user
+            continue
+        r = fapi.get_group(fcgroup)
+        fapi._check_response_code(r, 200)
+        fcgroup_data = r.json()
+        for admin in fcgroup_data['adminsEmails']:
+            yield admin
+        for member in fcgroup_data['membersEmails']:
+            yield member
+
 @fiss_cmd
 def meth_list(args):
     """ List workflows in the methods repository """
-    r = fapi.list_repository_methods(name=args.name)
+    r = fapi.list_repository_methods(namespace=args.namespace,
+                                     name=args.method,
+                                     snapshotId=args.snapshot_id)
     fapi._check_response_code(r, 200)
 
     # Parse the JSON for the workspace + namespace
@@ -481,6 +521,8 @@ def meth_list(args):
 @fiss_cmd
 def meth_exists(args):
     '''Determine whether a given workflow is present in methods repo'''
+    args.namespace = None
+    args.snapshot_id = None
     return len(meth_list(args)) != 0
 
 @fiss_cmd
@@ -519,7 +561,7 @@ def config_stop(args):
 
 @fiss_cmd
 def config_list(args):
-    """ List configurations in the methods repository or a workspace. """
+    """ List configuration(s) in the methods repository or a workspace. """
     verbose = fcconfig.verbosity
     if args.workspace:
         if verbose:
@@ -532,7 +574,9 @@ def config_list(args):
     else:
         if verbose:
             print("Retrieving method configs from method repository")
-        r = fapi.list_repository_configs()
+        r = fapi.list_repository_configs(namespace=args.namespace,
+                                         name=args.config,
+                                         snapshotId=args.snapshot_id)
         fapi._check_response_code(r, 200)
 
     # Parse the JSON for the workspace + namespace
@@ -563,6 +607,36 @@ def config_acl(args):
     acls = sorted(r.json(), key=lambda k: k['user'])
     return map(lambda acl: '{0}\t{1}'.format(acl['user'], acl['role']), acls)
 
+@fiss_cmd
+def config_set_acl(args):
+    """ Assign an ACL role to a list of users for a config. """
+    acl_updates = [{"user": user, "role": args.role} \
+                   for user in set(expand_fc_groups(args.users)) \
+                   if user != fapi.whoami()]
+
+    id = args.snapshot_id
+    if not id:
+        # get the latest snapshot_id for this method from the methods repo
+        r = fapi.list_repository_configs(namespace=args.namespace,
+                                         name=args.config)
+        fapi._check_response_code(r, 200)
+        versions = r.json()
+        if len(versions) == 0:
+            if fcconfig.verbosity:
+                eprint("Configuration {0}/{1} not found".format(args.namespace,
+                                                                args.config))
+            return 1
+        latest = sorted(versions, key=lambda c: c['snapshotId'])[-1]
+        id = latest['snapshotId']
+
+    r = fapi.update_repository_config_acl(args.namespace, args.config, id,
+                                          acl_updates)
+    fapi._check_response_code(r, 200)
+    if fcconfig.verbosity:
+        print("Updated ACL for {0}/{1}:{2}".format(args.namespace, args.config,
+                                                   id))
+    return 0
+
 @fiss_cmd
 def config_get(args):
     """ Retrieve a method config from a workspace, send stdout """
@@ -2093,22 +2167,31 @@ def main(argv=None):
     # List available methods
     subp = subparsers.add_parser('meth_list',
                                  description='List available workflows')
-    subp.add_argument('-n', '--name', default=None,required=False,
+    subp.add_argument('-m', '--method', default=None,
+                      help='name of single workflow to search for (optional)')
+    subp.add_argument('-n', '--namespace', default=None,
                       help='name of single workflow to search for (optional)')
+    subp.add_argument('-i', '--snapshot-id', default=None,
+                      help="Snapshot ID (version) of method/config")
     subp.set_defaults(func=meth_list)
 
     subp = subparsers.add_parser('meth_exists',
         description='Determine if named workflow exists in method repository')
-    subp.add_argument('name', help='name of method to search for In repository')
+    subp.add_argument('method', help='name of method to search for in repository')
     subp.set_defaults(func=meth_exists)
 
     # Configuration: list
     subp = subparsers.add_parser(
         'config_list', description='List available configurations')
-    subp.add_argument('-w', '--workspace', help='Workspace name',
-                      default=fcconfig.workspace, required=False)
+    subp.add_argument('-w', '--workspace', help='Workspace name')
     subp.add_argument('-p', '--project', default=fcconfig.project,
-                      help=proj_help, required=proj_required)
+                      help=proj_help)
+    subp.add_argument('-c', '--config', default=None,
+                      help='name of single workflow to search for (optional)')
+    subp.add_argument('-n', '--namespace', default=None,
+                      help='name of single workflow to search for (optional)')
+    subp.add_argument('-i', '--snapshot-id', default=None,
+                      help="Snapshot ID (version) of config (optional)")
     subp.set_defaults(func=config_list)
 
     # Configuration: delete
@@ -2200,16 +2283,12 @@ def main(argv=None):
     #        pushed into a separate function and/or auto-generated
 
     # Set ACL
-    # cacl_parser = subparsers.add_parser('config_set_acl',
-    #                           description='Set roles for config')
-    # cacl_parser.add_argument('namespace', help='Method namespace')
-    # cacl_parser.add_argument('name', help='Config name')
-    # cacl_parser.add_argument('snapshot_id', help='Snapshot ID')
-    # cacl_parser.add_argument('role', help='ACL role',
-    #                      choices=['OWNER', 'READER', 'WRITER', 'NO ACCESS'])
-    # cacl_parser.add_argument('users', metavar='user', help='Firecloud username',
-    #                          nargs='+')
-    # cacl_parser.set_defaults(func=meth_set_acl)
+    subp = subparsers.add_parser('config_set_acl', description='Assign an ' +
+                                 'ACL role to a list of users for a  config',
+                                 parents=[conf_parent, acl_parent])
+    subp.add_argument('-i', '--snapshot-id',
+                      help="Snapshot ID (version) of method/config")
+    subp.set_defaults(func=config_set_acl)
 
     # Status
     subp = subparsers.add_parser('health',