From 95e03f1274a39b8abf60648b8b9429dc2ad2ee53 Mon Sep 17 00:00:00 2001 From: David Heiman Date: Fri, 23 Sep 2022 22:20:45 -0400 Subject: [PATCH] v0.16.34 (#180) api.py * check for presence of clock skew argument rather than check google.auth.__version__ fiss.py * meth_acl/config_acl - changed map(f(x),iterable) returns into list comprehensions to make py2/3 consistent * mop updated to account for change in Terra output directory structure fccore.py * removed version string parser in favor of above check for the clock skew argument --- changelog.txt | 6 ++++++ firecloud/__about__.py | 2 +- firecloud/api.py | 11 +++++++++-- firecloud/fccore.py | 43 ------------------------------------------ firecloud/fiss.py | 24 +++++++++++++++++------ 5 files changed, 34 insertions(+), 52 deletions(-) diff --git a/changelog.txt b/changelog.txt index 1713a4a..fae2438 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,6 +3,12 @@ Change Log for FISSFC: the (Fi)recloud (S)ervice (S)elector ======================================================================= Terms used below: HL = high level interface, LL = low level interface +v0.16.34 - Hotfixes: in Python 2 map() returns a list, in Python 3, map() + returns a generator object - tasks that returned map(f(x),iterable) + now return a list comprehension; clock-skew now checks for argument + presence at runtime rather than google.auth version; mop HL function + updated to account for new output path structure in Terra. + v0.16.33 - HL: added space_size, space_cost; hotfixes: fixed Python 2 compatibility; blocked google-auth versions with restrictive clock-skew and enabled later versions with modifiable clock-skew, increasing clock-skew diff --git a/firecloud/__about__.py b/firecloud/__about__.py index de0f607..d3c40fd 100644 --- a/firecloud/__about__.py +++ b/firecloud/__about__.py @@ -1,2 +1,2 @@ # Package version -__version__ = "0.16.33" +__version__ = "0.16.34" diff --git a/firecloud/api.py b/firecloud/api.py index 99ba10c..16b9925 100755 --- a/firecloud/api.py +++ b/firecloud/api.py @@ -21,6 +21,8 @@ from six.moves.urllib.parse import urlencode, urljoin from six import string_types +import inspect + import google.auth from google.auth.exceptions import DefaultCredentialsError, RefreshError from google.auth.transport.requests import AuthorizedSession, Request @@ -28,7 +30,6 @@ from firecloud.errors import FireCloudServerError from firecloud.fccore import __fcconfig as fcconfig -from firecloud.fccore import release_tuple_from_version_string from firecloud.__about__ import __version__ FISS_USER_AGENT = "FISS/" + __version__ @@ -49,12 +50,18 @@ def _set_session(): global __USER_ID if __SESSION is None: + # determine if clock_skew_in_seconds is a parameter for id_token.verify_oauth2_token() + try: # PY3 + argspec = inspect.getfullargspec(id_token.verify_oauth2_token) + except AttributeError: # PY2 + argspec = inspect.getargspec(id_token.verify_oauth2_token) + try: __SESSION = AuthorizedSession(google.auth.default(['https://www.googleapis.com/auth/userinfo.profile', 'https://www.googleapis.com/auth/userinfo.email'])[0]) health() # google.auth 2.1.0 introduced a restrictive clock skew that was unmodifiable until 2.3.2 - if release_tuple_from_version_string(google.auth.__version__) >= (2,3,2): + if 'clock_skew_in_seconds' in argspec.args: __USER_ID = id_token.verify_oauth2_token(__SESSION.credentials.id_token, Request(session=__SESSION), clock_skew_in_seconds=10)['email'] diff --git a/firecloud/fccore.py b/firecloud/fccore.py index 6afdf8d..48c18e4 100644 --- a/firecloud/fccore.py +++ b/firecloud/fccore.py @@ -21,7 +21,6 @@ import tempfile import shutil import subprocess -import re from io import IOBase from firecloud import __about__ from google.auth import environment_vars @@ -220,46 +219,4 @@ def edit_file(name, backup=None): current_tolm = os.stat(name).st_mtime return current_tolm != previous_tolm - -# From PEP-440: -# https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions -VERSION_PATTERN = r""" - v? - (?: - (?:(?P[0-9]+)!)? # epoch - (?P[0-9]+(?:\.[0-9]+)*) # release segment - (?P
                                          # pre-release
-            [-_\.]?
-            (?P(a|b|c|rc|alpha|beta|pre|preview))
-            [-_\.]?
-            (?P[0-9]+)?
-        )?
-        (?P                                         # post release
-            (?:-(?P[0-9]+))
-            |
-            (?:
-                [-_\.]?
-                (?Ppost|rev|r)
-                [-_\.]?
-                (?P[0-9]+)?
-            )
-        )?
-        (?P                                          # dev release
-            [-_\.]?
-            (?Pdev)
-            [-_\.]?
-            (?P[0-9]+)?
-        )?
-    )
-    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
-"""
-
-version_regex = re.compile(
-    r"^\s*" + VERSION_PATTERN + r"\s*$",
-    re.VERBOSE | re.IGNORECASE,
-)
-
-def release_tuple_from_version_string(version_string):
-    return tuple(int(val) for val in version_regex.match(version_string).group('release').split('.'))
-
 # }}}
diff --git a/firecloud/fiss.py b/firecloud/fiss.py
index 4e1f598..526810d 100644
--- a/firecloud/fiss.py
+++ b/firecloud/fiss.py
@@ -476,7 +476,7 @@ def meth_acl(args):
                                                     args.snapshot_id)
     fapi._check_response_code(r, 200)
     acls = sorted(r.json(), key=lambda k: k['user'])
-    return map(lambda acl: '{0}\t{1}'.format(acl['user'], acl['role']), acls)
+    return ['{0}\t{1}'.format(acl['user'], acl['role']) for acl in acls]
 
 @fiss_cmd
 def meth_set_acl(args):
@@ -668,7 +668,7 @@ def config_acl(args):
                                                     args.snapshot_id)
     fapi._check_response_code(r, 200)
     acls = sorted(r.json(), key=lambda k: k['user'])
-    return map(lambda acl: '{0}\t{1}'.format(acl['user'], acl['role']), acls)
+    return ['{0}\t{1}'.format(acl['user'], acl['role']) for acl in acls]
 
 @fiss_cmd
 def config_set_acl(args):
@@ -1374,10 +1374,17 @@ def list_blob_gen(bucket_name):
 
         # Check to see if bucket file path contain the user's submission id
         # to ensure deletion of files in the submission directories only.
-        # Splits the bucket file: "gs://bucket_Id/submission_id/file_path", by the '/' symbol
-        # and stores values in a 5 length array: ['gs:', '' , 'bucket_Id', submission_id, file_path]
-        # to extract the submission id from the 4th element (index 3) of the array
-        bucket_files = set(bucket_file for bucket_file in bucket_file_sizes if bucket_file.split('/', 4)[3] in submission_ids)
+        # Splits the bucket file: gs://// or
+        #                         gs:///submissions//, by the '/' symbol
+        # and stores values in a 6 length array: ['gs:', '' , , , , ] or
+        #                                        ['gs:', '' , , 'submissions', , ]
+        # to extract the submission id from the 4th or 5th element (index 3 or 4) of the array
+        bucket_files = set()
+        for bucket_file in bucket_file_sizes:
+            for sub_id in bucket_file.split('/', 5)[3:5]:
+                if sub_id in submission_ids:
+                    bucket_files.add(bucket_file)
+                    break
         
     except Exception as e:
         eprint("Error retrieving files from bucket:" +
@@ -1428,12 +1435,17 @@ def can_delete(f):
             return False
         if filename == "rc":
             return False
+        if filename == "memory_retry_rc":
+            return False
         # Don't delete tool's exec.sh or script
         if filename in ('exec.sh', 'script'):
             return False
         # keep stdout, stderr, and output
         if filename in ('stderr', 'stdout', 'output'):
             return False
+        # Don't delete utility scripts
+        if filename in ('gcs_localization.sh', 'gcs_delocalization.sh', 'gcs_transfer.sh'):
+            return False
         # Only delete specified unreferenced files
         if args.include:
             for glob in args.include: