From 50a3e2a47a63e5249df49580b1d82abf2be20fdd Mon Sep 17 00:00:00 2001 From: David Heiman Date: Fri, 9 Sep 2022 09:16:39 -0400 Subject: [PATCH] v0.16.33 (#178) fccore.py: * Added function to extract release tuple from a standard version string (based on PEP 440) api.py: * parameter added to allow for clock skew in Oauth call * added version check of google.auth to determine availability of clock skew parameter fiss.py: * added space_size() * added space_cost() * removed Python 3 syntax that broke Python 2 compatibility setup.py: * blocked versions of google.auth with restrictive clock skew defaults and no way to modify them (2.1.0-2.3.1) Co-authored-by: Oliver Priebe <40225301+oliverpriebe@users.noreply.github.com> --- changelog.txt | 5 +++++ firecloud/__about__.py | 2 +- firecloud/api.py | 13 ++++++++++--- firecloud/fccore.py | 43 ++++++++++++++++++++++++++++++++++++++++++ firecloud/fiss.py | 27 +++++++++++++++++++++++++- setup.py | 2 +- 6 files changed, 86 insertions(+), 6 deletions(-) diff --git a/changelog.txt b/changelog.txt index ccf2abc..1713a4a 100644 --- a/changelog.txt +++ b/changelog.txt @@ -3,6 +3,11 @@ Change Log for FISSFC: the (Fi)recloud (S)ervice (S)elector ======================================================================= Terms used below: HL = high level interface, LL = low level interface +v0.16.33 - HL: added space_size, space_cost; hotfixes: fixed Python 2 compatibility; + blocked google-auth versions with restrictive clock-skew and enabled + later versions with modifiable clock-skew, increasing clock-skew + when appropriate. + v0.16.32 - Changed Dockerfile base image to python 3.10; fixed Python 3.10 incompatibility issues; pylint set to minimum version 2.0.0; LL: new new functions: delete_entities_of_type, rename_entity, diff --git a/firecloud/__about__.py b/firecloud/__about__.py index 792deb4..de0f607 100644 --- a/firecloud/__about__.py +++ b/firecloud/__about__.py @@ -1,2 +1,2 @@ # Package version -__version__ = "0.16.32" +__version__ = "0.16.33" diff --git a/firecloud/api.py b/firecloud/api.py index 6a990e1..99ba10c 100755 --- a/firecloud/api.py +++ b/firecloud/api.py @@ -28,6 +28,7 @@ from firecloud.errors import FireCloudServerError from firecloud.fccore import __fcconfig as fcconfig +from firecloud.fccore import release_tuple_from_version_string from firecloud.__about__ import __version__ FISS_USER_AGENT = "FISS/" + __version__ @@ -52,8 +53,14 @@ def _set_session(): __SESSION = AuthorizedSession(google.auth.default(['https://www.googleapis.com/auth/userinfo.profile', 'https://www.googleapis.com/auth/userinfo.email'])[0]) health() - __USER_ID = id_token.verify_oauth2_token(__SESSION.credentials.id_token, - Request(session=__SESSION))['email'] + # google.auth 2.1.0 introduced a restrictive clock skew that was unmodifiable until 2.3.2 + if release_tuple_from_version_string(google.auth.__version__) >= (2,3,2): + __USER_ID = id_token.verify_oauth2_token(__SESSION.credentials.id_token, + Request(session=__SESSION), + clock_skew_in_seconds=10)['email'] + else: + __USER_ID = id_token.verify_oauth2_token(__SESSION.credentials.id_token, + Request(session=__SESSION))['email'] except AttributeError: __USER_ID = __SESSION.credentials.service_account_email except (DefaultCredentialsError, RefreshError) as gae: @@ -1364,7 +1371,7 @@ def get_storage_cost(namespace, workspace): namespace (str): project to which workspace belongs workspace (str): Workspace name Swagger: - https://api.firecloud.org/#!/Workspaces/storageCostEstimate + https://api.firecloud.org/#/Workspaces/getStorageCostEstimate """ uri = "workspaces/{0}/{1}/storageCostEstimate".format(namespace, workspace) return __get(uri) diff --git a/firecloud/fccore.py b/firecloud/fccore.py index 48c18e4..6afdf8d 100644 --- a/firecloud/fccore.py +++ b/firecloud/fccore.py @@ -21,6 +21,7 @@ import tempfile import shutil import subprocess +import re from io import IOBase from firecloud import __about__ from google.auth import environment_vars @@ -219,4 +220,46 @@ def edit_file(name, backup=None): current_tolm = os.stat(name).st_mtime return current_tolm != previous_tolm + +# From PEP-440: +# https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions +VERSION_PATTERN = r""" + v? + (?: + (?:(?P[0-9]+)!)? # epoch + (?P[0-9]+(?:\.[0-9]+)*) # release segment + (?P
                                          # pre-release
+            [-_\.]?
+            (?P(a|b|c|rc|alpha|beta|pre|preview))
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+        (?P                                         # post release
+            (?:-(?P[0-9]+))
+            |
+            (?:
+                [-_\.]?
+                (?Ppost|rev|r)
+                [-_\.]?
+                (?P[0-9]+)?
+            )
+        )?
+        (?P                                          # dev release
+            [-_\.]?
+            (?Pdev)
+            [-_\.]?
+            (?P[0-9]+)?
+        )?
+    )
+    (?:\+(?P[a-z0-9]+(?:[-_\.][a-z0-9]+)*))?       # local version
+"""
+
+version_regex = re.compile(
+    r"^\s*" + VERSION_PATTERN + r"\s*$",
+    re.VERBOSE | re.IGNORECASE,
+)
+
+def release_tuple_from_version_string(version_string):
+    return tuple(int(val) for val in version_regex.match(version_string).group('release').split('.'))
+
 # }}}
diff --git a/firecloud/fiss.py b/firecloud/fiss.py
index 0a16734..4e1f598 100644
--- a/firecloud/fiss.py
+++ b/firecloud/fiss.py
@@ -119,6 +119,20 @@ def space_info(args):
     fapi._check_response_code(r, 200)
     return r.text
 
+@fiss_cmd
+def space_size(args):
+    """ Get storage size of a workspace. """
+    r = fapi.get_bucket_usage(args.project, args.workspace)
+    fapi._check_response_code(r, 200)
+    return r.text
+
+@fiss_cmd
+def space_cost(args):
+    """ Get average monthly storage cost of a workspace. """
+    r = fapi.get_storage_cost(args.project, args.workspace)
+    fapi._check_response_code(r, 200)
+    return r.text
+
 @fiss_cmd
 def space_delete(args):
     """ Delete a workspace. """
@@ -1320,7 +1334,7 @@ def update_referenced_files(referenced_files, attrs, bucket_prefix):
                             bucket_prefix)
 
     ## Now list files present in the bucket
-    def list_blob_gen(bucket_name: str):
+    def list_blob_gen(bucket_name):
         """Generate the list of blobs in the bucket and size of each blob
 
         Args:
@@ -2281,6 +2295,17 @@ def main(argv=None):
                                  description='Show workspace information')
     subp.set_defaults(func=space_info)
 
+    # Get workspace size
+    subp = subparsers.add_parser('space_size', parents=[workspace_parent],
+                                 description='Show workspace bucket usage')
+    subp.set_defaults(func=space_size)
+
+    # Get workspace monthly cost
+    subp = subparsers.add_parser('space_cost', parents=[workspace_parent],
+                                 description='Show workspace average monthly' +
+                                 ' cost')
+    subp.set_defaults(func=space_cost)
+
     # List workspaces
     subp = subparsers.add_parser('space_list',
             description='List available workspaces in projects (namespaces) ' +
diff --git a/setup.py b/setup.py
index e03cb37..6e0e7fa 100644
--- a/setup.py
+++ b/setup.py
@@ -142,7 +142,7 @@ def get_outputs(self):
     },
     test_suite = 'nose.collector',
     install_requires = [
-        'google-auth>=1.6.3',
+        'google-auth>=1.6.3,!=2.1.*,!=2.2.*,!=2.3.0,!=2.3.1',
         'google-cloud-storage>=1.36.1',
         'pydot',
         'requests[security]',