From e5531a22fc2870b6a2cc9747953d76062ccca6b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Alan=20H=C3=B6ng?= Date: Fri, 11 Sep 2020 14:10:31 +0200 Subject: [PATCH] Apply black formatting on whole repository --- drfs/__init__.py | 8 +- drfs/_version.py | 154 ++++++++------ drfs/agnostic.py | 6 +- drfs/filesystems/azure_blob.py | 14 +- drfs/filesystems/gcs.py | 7 +- drfs/filesystems/local.py | 18 +- drfs/filesystems/memory.py | 13 +- drfs/filesystems/s3.py | 6 +- drfs/filesystems/util.py | 2 +- drfs/luigi.py | 5 +- drfs/path.py | 27 ++- drfs/settings.py | 2 +- drfs/structure.py | 20 +- drfs/tests/conftest.py | 8 +- drfs/tests/filesystems/test_azure_blob.py | 8 +- .../tests/filesystems/test_azure_data_lake.py | 38 ++-- drfs/tests/filesystems/test_gcs.py | 1 + drfs/tests/filesystems/test_local.py | 47 +++-- drfs/tests/filesystems/test_s3.py | 22 +- drfs/tests/test_filesystem.py | 191 +++++++++--------- drfs/tests/test_path.py | 65 +++--- drfs/tests/test_structure.py | 11 +- drfs/util.py | 2 +- 23 files changed, 356 insertions(+), 319 deletions(-) diff --git a/drfs/__init__.py b/drfs/__init__.py index f3f4e31..9ab3c83 100644 --- a/drfs/__init__.py +++ b/drfs/__init__.py @@ -2,13 +2,15 @@ """Top-level package for drfs.""" __author__ = """Data Revenue GmbH""" -__email__ = 'alan@datarevenue.com' +__email__ = "alan@datarevenue.com" import confuse -config = confuse.LazyConfig('drfs', __name__) + +config = confuse.LazyConfig("drfs", __name__) from ._version import get_versions -__version__ = get_versions()['version'] + +__version__ = get_versions()["version"] del get_versions from .path import DRPath diff --git a/drfs/_version.py b/drfs/_version.py index 70992b1..c1fd65d 100644 --- a/drfs/_version.py +++ b/drfs/_version.py @@ -1,4 +1,3 @@ - # This file helps to compute a version number in source trees obtained from # git-archive tarball (such as those provided by githubs download-from-tag # feature). Distribution tarballs (built by setup.py sdist) and build @@ -58,17 +57,18 @@ class NotThisMethod(Exception): def register_vcs_handler(vcs, method): # decorator """Decorator to mark a method as the handler for a particular VCS.""" + def decorate(f): """Store f in HANDLERS[vcs][method].""" if vcs not in HANDLERS: HANDLERS[vcs] = {} HANDLERS[vcs][method] = f return f + return decorate -def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, - env=None): +def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=None): """Call the given command(s).""" assert isinstance(commands, list) p = None @@ -76,10 +76,13 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, try: dispcmd = str([c] + args) # remember shell=False, so use git.cmd on windows, not just git - p = subprocess.Popen([c] + args, cwd=cwd, env=env, - stdout=subprocess.PIPE, - stderr=(subprocess.PIPE if hide_stderr - else None)) + p = subprocess.Popen( + [c] + args, + cwd=cwd, + env=env, + stdout=subprocess.PIPE, + stderr=(subprocess.PIPE if hide_stderr else None), + ) break except EnvironmentError: e = sys.exc_info()[1] @@ -116,16 +119,22 @@ def versions_from_parentdir(parentdir_prefix, root, verbose): for i in range(3): dirname = os.path.basename(root) if dirname.startswith(parentdir_prefix): - return {"version": dirname[len(parentdir_prefix):], - "full-revisionid": None, - "dirty": False, "error": None, "date": None} + return { + "version": dirname[len(parentdir_prefix) :], + "full-revisionid": None, + "dirty": False, + "error": None, + "date": None, + } else: rootdirs.append(root) root = os.path.dirname(root) # up a level if verbose: - print("Tried directories %s but none started with prefix %s" % - (str(rootdirs), parentdir_prefix)) + print( + "Tried directories %s but none started with prefix %s" + % (str(rootdirs), parentdir_prefix) + ) raise NotThisMethod("rootdir doesn't start with parentdir_prefix") @@ -181,7 +190,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG):] for r in refs if r.startswith(TAG)]) + tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -190,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "master". - tags = set([r for r in refs if re.search(r'\d', r)]) + tags = set([r for r in refs if re.search(r"\d", r)]) if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -198,19 +207,26 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): for ref in sorted(tags): # sorting will prefer e.g. "2.0" over "2.0rc1" if ref.startswith(tag_prefix): - r = ref[len(tag_prefix):] + r = ref[len(tag_prefix) :] if verbose: print("picking %s" % r) - return {"version": r, - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": None, - "date": date} + return { + "version": r, + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": None, + "date": date, + } # no suitable tags, so version is "0+unknown", but full hex is still there if verbose: print("no suitable tags, using unknown + full revision id") - return {"version": "0+unknown", - "full-revisionid": keywords["full"].strip(), - "dirty": False, "error": "no suitable tags", "date": None} + return { + "version": "0+unknown", + "full-revisionid": keywords["full"].strip(), + "dirty": False, + "error": "no suitable tags", + "date": None, + } @register_vcs_handler("git", "pieces_from_vcs") @@ -225,8 +241,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if sys.platform == "win32": GITS = ["git.cmd", "git.exe"] - out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, - hide_stderr=True) + out, rc = run_command(GITS, ["rev-parse", "--git-dir"], cwd=root, hide_stderr=True) if rc != 0: if verbose: print("Directory %s not under git control" % root) @@ -234,10 +249,19 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): # if there is a tag matching tag_prefix, this yields TAG-NUM-gHEX[-dirty] # if there isn't one, this yields HEX[-dirty] (no NUM) - describe_out, rc = run_command(GITS, ["describe", "--tags", "--dirty", - "--always", "--long", - "--match", "%s*" % tag_prefix], - cwd=root) + describe_out, rc = run_command( + GITS, + [ + "describe", + "--tags", + "--dirty", + "--always", + "--long", + "--match", + "%s*" % tag_prefix, + ], + cwd=root, + ) # --long was added in git-1.5.5 if describe_out is None: raise NotThisMethod("'git describe' failed") @@ -260,17 +284,16 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): dirty = git_describe.endswith("-dirty") pieces["dirty"] = dirty if dirty: - git_describe = git_describe[:git_describe.rindex("-dirty")] + git_describe = git_describe[: git_describe.rindex("-dirty")] # now we have TAG-NUM-gHEX or HEX if "-" in git_describe: # TAG-NUM-gHEX - mo = re.search(r'^(.+)-(\d+)-g([0-9a-f]+)$', git_describe) + mo = re.search(r"^(.+)-(\d+)-g([0-9a-f]+)$", git_describe) if not mo: # unparseable. Maybe git-describe is misbehaving? - pieces["error"] = ("unable to parse git-describe output: '%s'" - % describe_out) + pieces["error"] = "unable to parse git-describe output: '%s'" % describe_out return pieces # tag @@ -279,10 +302,12 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = ("tag '%s' doesn't start with prefix '%s'" - % (full_tag, tag_prefix)) + pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + full_tag, + tag_prefix, + ) return pieces - pieces["closest-tag"] = full_tag[len(tag_prefix):] + pieces["closest-tag"] = full_tag[len(tag_prefix) :] # distance: number of commits since tag pieces["distance"] = int(mo.group(2)) @@ -293,13 +318,13 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): else: # HEX: no tags pieces["closest-tag"] = None - count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], - cwd=root) + count_out, rc = run_command(GITS, ["rev-list", "HEAD", "--count"], cwd=root) pieces["distance"] = int(count_out) # total number of commits # commit date: see ISO-8601 comment in git_versions_from_keywords() - date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], - cwd=root)[0].strip() + date = run_command(GITS, ["show", "-s", "--format=%ci", "HEAD"], cwd=root)[ + 0 + ].strip() pieces["date"] = date.strip().replace(" ", "T", 1).replace(" ", "", 1) return pieces @@ -330,8 +355,7 @@ def render_pep440(pieces): rendered += ".dirty" else: # exception #1 - rendered = "0+untagged.%d.g%s" % (pieces["distance"], - pieces["short"]) + rendered = "0+untagged.%d.g%s" % (pieces["distance"], pieces["short"]) if pieces["dirty"]: rendered += ".dirty" return rendered @@ -445,11 +469,13 @@ def render_git_describe_long(pieces): def render(pieces, style): """Render the given version pieces into the requested style.""" if pieces["error"]: - return {"version": "unknown", - "full-revisionid": pieces.get("long"), - "dirty": None, - "error": pieces["error"], - "date": None} + return { + "version": "unknown", + "full-revisionid": pieces.get("long"), + "dirty": None, + "error": pieces["error"], + "date": None, + } if not style or style == "default": style = "pep440" # the default @@ -469,9 +495,13 @@ def render(pieces, style): else: raise ValueError("unknown style '%s'" % style) - return {"version": rendered, "full-revisionid": pieces["long"], - "dirty": pieces["dirty"], "error": None, - "date": pieces.get("date")} + return { + "version": rendered, + "full-revisionid": pieces["long"], + "dirty": pieces["dirty"], + "error": None, + "date": pieces.get("date"), + } def get_versions(): @@ -485,8 +515,7 @@ def get_versions(): verbose = cfg.verbose try: - return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, - verbose) + return git_versions_from_keywords(get_keywords(), cfg.tag_prefix, verbose) except NotThisMethod: pass @@ -495,13 +524,16 @@ def get_versions(): # versionfile_source is the relative path from the top of the source # tree (where the .git directory might live) to this file. Invert # this to find the root from __file__. - for i in cfg.versionfile_source.split('/'): + for i in cfg.versionfile_source.split("/"): root = os.path.dirname(root) except NameError: - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to find root of source tree", - "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to find root of source tree", + "date": None, + } try: pieces = git_pieces_from_vcs(cfg.tag_prefix, root, verbose) @@ -515,6 +547,10 @@ def get_versions(): except NotThisMethod: pass - return {"version": "0+unknown", "full-revisionid": None, - "dirty": None, - "error": "unable to compute version", "date": None} + return { + "version": "0+unknown", + "full-revisionid": None, + "dirty": None, + "error": "unable to compute version", + "date": None, + } diff --git a/drfs/agnostic.py b/drfs/agnostic.py index 0008e5a..419a890 100644 --- a/drfs/agnostic.py +++ b/drfs/agnostic.py @@ -25,8 +25,7 @@ def mv(src, dst, opts=None): def makedirs(path, *args, opts=None, **kwargs): """Filesystem-agnostic makedirs.""" - return get_fs(path, opts=opts) \ - .makedirs(path, *args, **kwargs) + return get_fs(path, opts=opts).makedirs(path, *args, **kwargs) def rmdir(path, opts=None): @@ -37,7 +36,8 @@ def rmdir(path, opts=None): def savefig(path, *args, opts=None, **kwargs): """Filesystem-agnostic savefig.""" import matplotlib.pyplot as plt + path = str(path) # in case it's pathlib object fmt = os.path.splitext(path)[1][1:] - with open(path, 'wb', opts=opts) as f: + with open(path, "wb", opts=opts) as f: plt.savefig(f, format=fmt, *args, **kwargs) diff --git a/drfs/filesystems/azure_blob.py b/drfs/filesystems/azure_blob.py index ab8c8a9..78c5e41 100644 --- a/drfs/filesystems/azure_blob.py +++ b/drfs/filesystems/azure_blob.py @@ -8,30 +8,30 @@ class AzureBlobFileSystem(FileSystemBase): fs_cls = abfs.DaskAzureBlobFileSystem - scheme = 'abfs' + scheme = "abfs" is_remote = True - + @allow_pathlib def exists(self, path, *args, **kwargs): - return self.fs.exists(*extract_abfs_parts(path)[1:], - *args, **kwargs) + return self.fs.exists(*extract_abfs_parts(path)[1:], *args, **kwargs) @return_pathlib @return_schemes @allow_pathlib def ls(self, path, *args, **kwargs): acc, cont, rest = extract_abfs_parts(path) - res = self.fs.ls(cont, os.path.join(rest, '*'), *args, **kwargs) + res = self.fs.ls(cont, os.path.join(rest, "*"), *args, **kwargs) return [os.path.join(acc, cont, item) for item in res] def extract_abfs_parts(path): import re - match = re.match('abfs://(.*?)/(.*?)/(.*)', path) + + match = re.match("abfs://(.*?)/(.*?)/(.*)", path) if match is None: raise ValueError(f"Path {path} doesn't match abfs path pattern.") account, container, rest = match.groups() return account, container, rest -FILESYSTEMS['abfs'] = AzureBlobFileSystem +FILESYSTEMS["abfs"] = AzureBlobFileSystem diff --git a/drfs/filesystems/gcs.py b/drfs/filesystems/gcs.py index b8b65be..0899d52 100644 --- a/drfs/filesystems/gcs.py +++ b/drfs/filesystems/gcs.py @@ -6,8 +6,9 @@ class GCSFileSystem(FileSystemBase): """Wrapper for dask's GCSFileSystem.""" + fs_cls = gcsfs.GCSFileSystem - scheme = 'gs' + scheme = "gs" is_remote = True @allow_pathlib @@ -19,5 +20,5 @@ def makedirs(self, *args, **kwargs): raise NotImplementedError -FILESYSTEMS['gs'] = GCSFileSystem -FILESYSTEMS['gcs'] = GCSFileSystem +FILESYSTEMS["gs"] = GCSFileSystem +FILESYSTEMS["gcs"] = GCSFileSystem diff --git a/drfs/filesystems/local.py b/drfs/filesystems/local.py index 950c02c..5cdf12b 100644 --- a/drfs/filesystems/local.py +++ b/drfs/filesystems/local.py @@ -14,7 +14,7 @@ class LocalFileSystem(FileSystemBase): """Emulates a remote filesystem on the local disk.""" fs_cls = None # we could do even without subclassing FSBase - scheme = '' + scheme = "" is_remote = False @allow_pathlib @@ -80,17 +80,15 @@ def rmdir(self, path): def info(self, path): """Get a dict with only LastModified time in UTC.""" mtime = os.path.getmtime(path) - return { - 'LastModified': datetime.datetime.fromtimestamp(mtime, pytz.UTC) - } + return {"LastModified": datetime.datetime.fromtimestamp(mtime, pytz.UTC)} @return_pathlib @allow_pathlib def walk(self, path): """Walk over all files in this directory (recursively).""" - return [os.path.join(root, f) - for root, dirs, files in os.walk(path) - for f in files] + return [ + os.path.join(root, f) for root, dirs, files in os.walk(path) for f in files + ] @return_pathlib @allow_pathlib @@ -100,8 +98,8 @@ def glob(self, path): @allow_pathlib def touch(self, path): - self.open(path, 'w').close() + self.open(path, "w").close() -FILESYSTEMS[''] = LocalFileSystem -FILESYSTEMS['file'] = LocalFileSystem +FILESYSTEMS[""] = LocalFileSystem +FILESYSTEMS["file"] = LocalFileSystem diff --git a/drfs/filesystems/memory.py b/drfs/filesystems/memory.py index 6c69247..4a02e2c 100644 --- a/drfs/filesystems/memory.py +++ b/drfs/filesystems/memory.py @@ -6,7 +6,7 @@ class MemoryFileSystem(FileSystemBase): fs_cls = memfs.MemoryFileSystem - scheme = 'memory' + scheme = "memory" is_remote = True supports_scheme = False @@ -45,21 +45,22 @@ def rm(self, path, recursive=False): def _recursive_rm(self, path): for res in self.fs.ls(path, detail=True): - if res['type'] == 'directory': - self._recursive_rm(res['name']) + if res["type"] == "directory": + self._recursive_rm(res["name"]) else: - self.fs.rm(res['name']) - + self.fs.rm(res["name"]) def put(self, filename, path, **kwargs): from drfs.path import asstr + filename, path = asstr(filename), asstr(path) return self.fs.put(filename, path, **kwargs) def get(self, path, filename, **kwargs): from drfs.path import asstr + path, filename = asstr(path), asstr(filename) return self.fs.get(path, filename, **kwargs) -FILESYSTEMS['memory'] = MemoryFileSystem +FILESYSTEMS["memory"] = MemoryFileSystem diff --git a/drfs/filesystems/s3.py b/drfs/filesystems/s3.py index b7550d1..229634c 100644 --- a/drfs/filesystems/s3.py +++ b/drfs/filesystems/s3.py @@ -6,7 +6,7 @@ class S3FileSystem(FileSystemBase): fs_cls = s3fs.S3FileSystem - scheme = 's3' + scheme = "s3" is_remote = True @allow_pathlib @@ -24,13 +24,15 @@ def rm(self, path, recursive=False, **kwargs): def put(self, filename, path, **kwargs): from drfs.path import asstr + filename, path = asstr(filename), asstr(path) return self.fs.put(filename, path, **kwargs) def get(self, path, filename, **kwargs): from drfs.path import asstr + path, filename = asstr(path), asstr(filename) return self.fs.get(path, filename, **kwargs) -FILESYSTEMS['s3'] = S3FileSystem +FILESYSTEMS["s3"] = S3FileSystem diff --git a/drfs/filesystems/util.py b/drfs/filesystems/util.py index 93b00b6..1179457 100644 --- a/drfs/filesystems/util.py +++ b/drfs/filesystems/util.py @@ -38,7 +38,7 @@ def get_fs(path, opts=None, rtype="instance"): f"{set(FILESYSTEMS.keys())}" ) config_scheme_key = protocol if protocol else "file" - opts_ = config['fs_opts'][config_scheme_key].get(dict) # type: dict + opts_ = config["fs_opts"][config_scheme_key].get(dict) # type: dict if opts is not None: opts_.update(opts) opts_ = _fix_opts_abfs(cls, path, opts_) diff --git a/drfs/luigi.py b/drfs/luigi.py index ba5a446..155006e 100644 --- a/drfs/luigi.py +++ b/drfs/luigi.py @@ -5,11 +5,10 @@ try: from luigi.target import FileSystemTarget except ImportError: - raise ImportError('Could not import luigi library. Try installing it.') + raise ImportError("Could not import luigi library. Try installing it.") class FileTarget(FileSystemTarget): - def __init__(self, path, **kwargs): """Target for any kind of storage. Infers file system automatically. @@ -26,7 +25,7 @@ def __init__(self, path, **kwargs): @property def fs(self): - return get_fs(self.path, opts=self.storage_options, rtype='instance') + return get_fs(self.path, opts=self.storage_options, rtype="instance") def open(self, *args, **kwargs): return self.fs.open(self.path, *args, **kwargs) diff --git a/drfs/path.py b/drfs/path.py index 2bb930e..688d438 100644 --- a/drfs/path.py +++ b/drfs/path.py @@ -16,15 +16,15 @@ class DRPathMixin: @property def is_template(self): s = str(self) - return 0 <= s.find('{') < s.find('}') + return 0 <= s.find("{") < s.find("}") @property def is_wildcard(self): - return '*' in str(self) + return "*" in str(self) @property def flag(self): - return self / '_SUCCESS' + return self / "_SUCCESS" def format(self, *args, **kwargs): return DRPath(str(self).format(*args, **kwargs)) @@ -79,8 +79,10 @@ def _accessor(self): try: self._acc_real = FILESYSTEMS[self.scheme](**self.opts) except KeyError: - raise ValueError('Scheme {} not found in available filesystems' - ', try installing it.'.format(self.scheme)) + raise ValueError( + "Scheme {} not found in available filesystems" + ", try installing it.".format(self.scheme) + ) return self._acc_real def exists(self): @@ -122,13 +124,18 @@ def path(self): """ # https://tools.ietf.org/html/rfc3986#appendix-A - safe_pchars = '-._~!$&\'()*+,;=:@{}%' + safe_pchars = "-._~!$&'()*+,;=:@{}%" begin = 1 if self._drv or self._root else 0 - return self._root \ - + self._flavour.sep.join(urllib.parse.quote(i, safe=safe_pchars) for i in self._parts[begin:-1] + [self.name]) \ - + self.trailing_sep + return ( + self._root + + self._flavour.sep.join( + urllib.parse.quote(i, safe=safe_pchars) + for i in self._parts[begin:-1] + [self.name] + ) + + self.trailing_sep + ) def _make_child(self, args): res = super()._make_child(args) @@ -144,7 +151,7 @@ class LocalPath(PATH_CLASS, DRPathMixin): class DRPath: def __new__(cls, path, *args, **kwargs): if cls is DRPath: - if get_fs(path, rtype='class').is_remote: + if get_fs(path, rtype="class").is_remote: cls = RemotePath else: cls = LocalPath diff --git a/drfs/settings.py b/drfs/settings.py index 0da5223..b3695a3 100644 --- a/drfs/settings.py +++ b/drfs/settings.py @@ -3,5 +3,5 @@ """ -GET_FILESYSTEM_RTYPE = 'class' +GET_FILESYSTEM_RTYPE = "class" FS_OPTS = {} diff --git a/drfs/structure.py b/drfs/structure.py index 965cb2c..ebd7616 100644 --- a/drfs/structure.py +++ b/drfs/structure.py @@ -10,6 +10,7 @@ def _root_function(value): def foo(self): return self.root / value + return foo @@ -17,7 +18,7 @@ class _MetaTree(type): def __new__(mcs, name, bases, attrs): new_attrs = {} for attr_name, attr_value in attrs.items(): - if attr_name in ['root'] or attr_name.startswith('__'): + if attr_name in ["root"] or attr_name.startswith("__"): new_attrs[attr_name] = attr_value elif isinstance(attr_value, (str, DRPathMixin)): new_attrs[attr_name] = property(_root_function(attr_value)) @@ -46,29 +47,26 @@ def root(self, value): if isinstance(node_value, Tree): node_value = copy(node_value) setattr(self, node_name, node_value) - node_root = getattr(node_value, '__root__', node_name) + node_root = getattr(node_value, "__root__", node_name) node_value.root = self._root / node_root def _get_nodes(self): - nodes = inspect.getmembers( - self, - lambda a: not (inspect.isroutine(a)), - ) - nodes = [n for n in nodes if not n[0].startswith('__')] + nodes = inspect.getmembers(self, lambda a: not (inspect.isroutine(a))) + nodes = [n for n in nodes if not n[0].startswith("__")] return nodes def __repr__(self): res = "" for node_name, node_value in self._get_nodes(): if isinstance(node_value, DRPathMixin): - if node_name == 'root': + if node_name == "root": continue - s = f'{node_name}: {node_value}\n' + s = f"{node_name}: {node_value}\n" elif isinstance(node_value, Tree): s = f'{node_name}:\n{indent(str(node_value), " ")}' else: - s = '' - res = f'{res}{s}' + s = "" + res = f"{res}{s}" return res def add(self, key, value): diff --git a/drfs/tests/conftest.py b/drfs/tests/conftest.py index 8a05b3c..7c7e350 100644 --- a/drfs/tests/conftest.py +++ b/drfs/tests/conftest.py @@ -8,9 +8,10 @@ def s3(): m.start() import boto3 import s3fs - conn = boto3.client('s3') - conn.create_bucket(Bucket='test-bucket') - conn.put_object(Bucket='test-bucket', Key='test.txt', Body=b'bla') + + conn = boto3.client("s3") + conn.create_bucket(Bucket="test-bucket") + conn.put_object(Bucket="test-bucket", Key="test.txt", Body=b"bla") yield s3fs.S3FileSystem(anon=False) m.stop() @@ -26,6 +27,7 @@ def s3_data_dir(): mock.start() import boto3 import s3fs + conn = boto3.client("s3") conn.create_bucket(Bucket="s3-test-bucket") fs = s3fs.S3FileSystem() diff --git a/drfs/tests/filesystems/test_azure_blob.py b/drfs/tests/filesystems/test_azure_blob.py index 875c90d..fedfaa4 100644 --- a/drfs/tests/filesystems/test_azure_blob.py +++ b/drfs/tests/filesystems/test_azure_blob.py @@ -7,9 +7,7 @@ def test_extract_abfs_parts(): - assert extract_abfs_parts('abfs://acc/cont/file') \ - == ('acc', 'cont', 'file') - assert extract_abfs_parts('abfs://acc/cont/dir/file') \ - == ('acc', 'cont', 'dir/file') + assert extract_abfs_parts("abfs://acc/cont/file") == ("acc", "cont", "file") + assert extract_abfs_parts("abfs://acc/cont/dir/file") == ("acc", "cont", "dir/file") with pytest.raises(ValueError, match="doesn't match abfs"): - extract_abfs_parts('abfas://acc/cont/dir/file') + extract_abfs_parts("abfas://acc/cont/dir/file") diff --git a/drfs/tests/filesystems/test_azure_data_lake.py b/drfs/tests/filesystems/test_azure_data_lake.py index 6b4f4d9..15f116d 100644 --- a/drfs/tests/filesystems/test_azure_data_lake.py +++ b/drfs/tests/filesystems/test_azure_data_lake.py @@ -9,45 +9,45 @@ def mock_azure_fs_native(monkeypatch): fs = MagicMock(spec=AzureDLFileSystem) fs.ls.return_value = [ - 'folder/directory/file.txt', - 'folder/directory/file2.txt', - 'folder/directory/file3.txt' + "folder/directory/file.txt", + "folder/directory/file2.txt", + "folder/directory/file3.txt", ] fs.glob.return_value = [ - 'folder/directory/file.txt', - 'folder/directory/file2.txt', - 'folder/directory/file3.txt' + "folder/directory/file.txt", + "folder/directory/file2.txt", + "folder/directory/file3.txt", ] fs.kwargs = {} cls = MagicMock() cls.return_value = fs - monkeypatch.setattr(azure_datalake, 'AzureDLFileSystem', cls) - monkeypatch.setattr(azure_datalake.lib, 'auth', lambda *args, **kwargs: 'token') + monkeypatch.setattr(azure_datalake, "AzureDLFileSystem", cls) + monkeypatch.setattr(azure_datalake.lib, "auth", lambda *args, **kwargs: "token") def test_custom_connect(): fs = azure_datalake.AzureDataLakeFileSystem() - path = fs._connect('adl://intvanprofi/some/path.txt') - assert fs.fs.kwargs['store_name'] == 'intvanprofi' - assert not path.startswith('adl://intvanprofi') + path = fs._connect("adl://intvanprofi/some/path.txt") + assert fs.fs.kwargs["store_name"] == "intvanprofi" + assert not path.startswith("adl://intvanprofi") def test_ls(): fs = azure_datalake.AzureDataLakeFileSystem() - res = fs.ls('adl://intvanprofi/some/path/to/directory') + res = fs.ls("adl://intvanprofi/some/path/to/directory") - fs.fs.ls.assert_called_once_with('/some/path/to/directory') + fs.fs.ls.assert_called_once_with("/some/path/to/directory") for p in res: - assert p.hostname == 'intvanprofi' - assert p.scheme == 'adl' + assert p.hostname == "intvanprofi" + assert p.scheme == "adl" def test_glob(): fs = azure_datalake.AzureDataLakeFileSystem() - res = fs.glob('adl://intvanprofi/some/path/to/*.csv') + res = fs.glob("adl://intvanprofi/some/path/to/*.csv") - fs.fs.glob.assert_called_once_with('/some/path/to/*.csv') + fs.fs.glob.assert_called_once_with("/some/path/to/*.csv") for p in res: - assert p.hostname == 'intvanprofi' - assert p.scheme == 'adl' + assert p.hostname == "intvanprofi" + assert p.scheme == "adl" diff --git a/drfs/tests/filesystems/test_gcs.py b/drfs/tests/filesystems/test_gcs.py index 73b71d7..96170d8 100644 --- a/drfs/tests/filesystems/test_gcs.py +++ b/drfs/tests/filesystems/test_gcs.py @@ -1,4 +1,5 @@ import pytest + try: from drfs.filesystems.gcs import GCSFileSystem except ImportError: diff --git a/drfs/tests/filesystems/test_local.py b/drfs/tests/filesystems/test_local.py index c0f7bf7..5212d05 100644 --- a/drfs/tests/filesystems/test_local.py +++ b/drfs/tests/filesystems/test_local.py @@ -8,26 +8,26 @@ def test_local_filesystem(tmpdir): - existing_file = tmpdir.join('some_file.txt') - non_existing_file = tmpdir.join('not_here') - with open(existing_file, 'w') as f: - f.write('test') + existing_file = tmpdir.join("some_file.txt") + non_existing_file = tmpdir.join("not_here") + with open(existing_file, "w") as f: + f.write("test") fs = LocalFileSystem() assert fs.exists(existing_file) assert not fs.exists(non_existing_file) assert fs.ls(tmpdir) == [existing_file] - assert 'LastModified' in fs.info(existing_file) - assert isinstance(fs.info(existing_file)['LastModified'], datetime) + assert "LastModified" in fs.info(existing_file) + assert isinstance(fs.info(existing_file)["LastModified"], datetime) with pytest.raises(FileNotFoundError): fs.info(non_existing_file) - with fs.open(existing_file, 'r') as f: - assert f.read() == 'test' + with fs.open(existing_file, "r") as f: + assert f.read() == "test" - copy_path = tmpdir.join('copy.txt') + copy_path = tmpdir.join("copy.txt") fs.copy(existing_file, copy_path) assert copy_path.exists() - copy_path2 = tmpdir.join('copy2.txt') + copy_path2 = tmpdir.join("copy2.txt") fs.cp(existing_file, copy_path2) assert copy_path2.exists() @@ -40,35 +40,35 @@ def test_local_filesystem(tmpdir): def test_glob_scheme_local(tmpdir): fs = LocalFileSystem() - fs.touch(tmpdir / '/test.txt') - fs.touch(tmpdir / '/test2.txt') - fs.touch(tmpdir / 'dir1' / 'deep_test.txt') + fs.touch(tmpdir / "/test.txt") + fs.touch(tmpdir / "/test2.txt") + fs.touch(tmpdir / "dir1" / "deep_test.txt") - res = fs.glob((tmpdir / '*').strpath) + res = fs.glob((tmpdir / "*").strpath) assert len(res) == 3 # 2 files + 1 dir assert all(isinstance(item, LocalPath) for item in res) - assert not any('deep_test' in str(item) for item in res) + assert not any("deep_test" in str(item) for item in res) def test_walk_scheme_local(tmpdir): fs = LocalFileSystem() - fs.touch(tmpdir / '/test.txt') - fs.touch(tmpdir / '/test2.txt') - fs.touch(tmpdir / 'dir1' / 'deep_test.txt') + fs.touch(tmpdir / "/test.txt") + fs.touch(tmpdir / "/test2.txt") + fs.touch(tmpdir / "dir1" / "deep_test.txt") res = fs.walk(tmpdir.strpath) assert len(res) == 3 # 3 files assert all(isinstance(item, Path) for item in res) - assert any('deep_test' in str(item) for item in res) - assert not any(str(item).rstrip('/').endswith('dir1') for item in res) + assert any("deep_test" in str(item) for item in res) + assert not any(str(item).rstrip("/").endswith("dir1") for item in res) def test_remove(tmpdir): fs = LocalFileSystem() - empty_dir = tmpdir / 'empty' + empty_dir = tmpdir / "empty" fs.makedirs(empty_dir) - dir1 = tmpdir / 'dir1' - file1 = dir1 / 'deep_test.txt' + dir1 = tmpdir / "dir1" + file1 = dir1 / "deep_test.txt" fs.touch(file1) assert fs.exists(empty_dir) @@ -82,4 +82,3 @@ def test_remove(tmpdir): fs.remove(dir1, recursive=True) assert not fs.exists(file1) assert not fs.exists(dir1) - diff --git a/drfs/tests/filesystems/test_s3.py b/drfs/tests/filesystems/test_s3.py index 2cab8d9..f13e5c1 100644 --- a/drfs/tests/filesystems/test_s3.py +++ b/drfs/tests/filesystems/test_s3.py @@ -10,13 +10,13 @@ def test_glob_scheme_s3(s3): fs = S3FileSystem() - fs.touch('s3://test-bucket/test2.txt') - fs.touch('s3://test-bucket/dir1/deep_test.txt') - res = fs.glob('s3://test-bucket/*') + fs.touch("s3://test-bucket/test2.txt") + fs.touch("s3://test-bucket/dir1/deep_test.txt") + res = fs.glob("s3://test-bucket/*") assert len(res) == 3 # 2 files + 1 dir assert all(isinstance(item, RemotePath) for item in res) - assert all(str(item).startswith('s3://') for item in res) - assert not any('deep_test' in str(item) for item in res) + assert all(str(item).startswith("s3://") for item in res) + assert not any("deep_test" in str(item) for item in res) # # also check walk here to check if it works after hacky stuff in glob # res = fs.walk('s3://test-bucket/') @@ -30,14 +30,14 @@ def test_glob_scheme_s3(s3): @pytest.mark.skip("Seems to be buggy. Do we need walk anyway?") def test_walk_scheme_s3(s3): fs = S3FileSystem() - fs.touch('s3://test-bucket/test2.txt') - fs.touch('s3://test-bucket/dir1/deep_test.txt') - res = fs.walk('s3://test-bucket/') + fs.touch("s3://test-bucket/test2.txt") + fs.touch("s3://test-bucket/dir1/deep_test.txt") + res = fs.walk("s3://test-bucket/") assert len(res) == 3 # 3 files assert all(isinstance(item, RemotePath) for item in res) - assert all(str(item).startswith('s3://') for item in res) - assert any('deep_test' in str(item) for item in res) - assert not any(str(item).rstrip('/').endswith('dir1') for item in res) + assert all(str(item).startswith("s3://") for item in res) + assert any("deep_test" in str(item) for item in res) + assert not any(str(item).rstrip("/").endswith("dir1") for item in res) def test_remove_recursive(s3): diff --git a/drfs/tests/test_filesystem.py b/drfs/tests/test_filesystem.py index dcef089..128d71c 100644 --- a/drfs/tests/test_filesystem.py +++ b/drfs/tests/test_filesystem.py @@ -21,82 +21,80 @@ try: from drfs.filesystems.s3 import S3FileSystem except ImportError: - warn('s3fs not installed, skipping filesystem tests depending on s3fs') + warn("s3fs not installed, skipping filesystem tests depending on s3fs") S3FileSystem = False try: from drfs.filesystems.gcs import GCSFileSystem + # my_vcr = GCSFSVCR(os.path.join(os.path.dirname(__file__), 'recordings'))\ # .my_vcr except ImportError: - warn('gcsfs not installed, skipping filesystem tests depending on gcsfs') + warn("gcsfs not installed, skipping filesystem tests depending on gcsfs") GCSFileSystem = False try: - from drfs.filesystems.azure_blob import AzureBlobFileSystem, \ - extract_abfs_parts + from drfs.filesystems.azure_blob import AzureBlobFileSystem, extract_abfs_parts except ImportError: - warn('azureblobfs not installed, skipping filesystem tests depending on ' - 'it.') + warn("azureblobfs not installed, skipping filesystem tests depending on " "it.") AzureBlobFileSystem = False -EXPECTED_METHODS = {'remove', 'ls', 'open', 'exists', 'mv', 'makedirs', - 'rmdir'} +EXPECTED_METHODS = {"remove", "ls", "open", "exists", "mv", "makedirs", "rmdir"} def _get_fs_tuples(): res = [ - ('user/some_file.txt', LocalFileSystem), - ('file://user/some_file.txt', LocalFileSystem), - ('/home/user/some_file.txt', LocalFileSystem), + ("user/some_file.txt", LocalFileSystem), + ("file://user/some_file.txt", LocalFileSystem), + ("/home/user/some_file.txt", LocalFileSystem), (r"C:\Users\alhoeng\drfs", LocalFileSystem), - ('memory://home/directory/some_file.txt', MemoryFileSystem), + ("memory://home/directory/some_file.txt", MemoryFileSystem), ] if S3FileSystem: - res.append( - ('s3://test-bucket/some_file.txt', S3FileSystem)) + res.append(("s3://test-bucket/some_file.txt", S3FileSystem)) if GCSFileSystem: - res.extend([ - ('gs://test-bucket/some_file.txt', GCSFileSystem), - ('gcs://test-bucket/some_file.txt', GCSFileSystem) - ]) + res.extend( + [ + ("gs://test-bucket/some_file.txt", GCSFileSystem), + ("gcs://test-bucket/some_file.txt", GCSFileSystem), + ] + ) if AzureBlobFileSystem: - res.extend([ - ('abfs://test-account/test-container/some_file.txt', - AzureBlobFileSystem), - ]) + res.extend( + [("abfs://test-account/test-container/some_file.txt", AzureBlobFileSystem)] + ) return res -@pytest.mark.skip('Non-deterministic behaviour of moto. Sometimes this fails. ' - 'Should be investigated and fixed.') +@pytest.mark.skip( + "Non-deterministic behaviour of moto. Sometimes this fails. " + "Should be investigated and fixed." +) def test_remote_path(s3): fs = s3 - assert fs.open('s3://test-bucket/test.txt', 'rb').read() - p = RemotePath('s3://test-bucket/test.txt') + assert fs.open("s3://test-bucket/test.txt", "rb").read() + p = RemotePath("s3://test-bucket/test.txt") - with p.open('rb') as fp: - assert fp.read() == b'bla' + with p.open("rb") as fp: + assert fp.read() == b"bla" - with p.with_name('test2.txt').open('wb') as fp: - fp.write(b'bla') + with p.with_name("test2.txt").open("wb") as fp: + fp.write(b"bla") - assert str(next(p.parent.iterdir())) == 's3://test-bucket/test.txt' + assert str(next(p.parent.iterdir())) == "s3://test-bucket/test.txt" assert p.exists() assert p.parent.exists() p.unlink() - assert list(p.parent.iterdir()) == \ - [RemotePath('s3://test-bucket/test2.txt')] + assert list(p.parent.iterdir()) == [RemotePath("s3://test-bucket/test2.txt")] assert not p.exists() def test_filesystem_attributes(): for cls in FILESYSTEMS.values(): for key in EXPECTED_METHODS: - assert hasattr(cls, key) and \ - callable(getattr(cls, key)) + assert hasattr(cls, key) and callable(getattr(cls, key)) -@pytest.mark.parametrize('path, fs', _get_fs_tuples()) +@pytest.mark.parametrize("path, fs", _get_fs_tuples()) def test_get_fs(path, fs): assert isinstance(get_fs(path), fs) @@ -104,43 +102,47 @@ def test_get_fs(path, fs): def relative_path_factory(root): def f(arg): arg = str(arg) - return arg.replace(root, '') + return arg.replace(root, "") + return f -@pytest.mark.usefixtures('s3') +@pytest.mark.usefixtures("s3") def test_filesystems(tmpdir, s3): fs = LocalFileSystem() s3 = S3FileSystem() - bucket = RemotePath('s3://test-bucket') + bucket = RemotePath("s3://test-bucket") tmpdir = Path(str(tmpdir)) norm_local = relative_path_factory(str(tmpdir)) # Should it be s3://test-bucket - norm_s3 = relative_path_factory('s3://test-bucket') - - with tmpdir.joinpath('test.txt').open(mode='w') as fp: - fp.write('bla') - - assert list(map(norm_local, fs.ls(tmpdir))) == \ - list(map(norm_s3, s3.ls(bucket))) - - assert list(map(norm_local, fs.glob(tmpdir.joinpath('*.txt')))) == \ - list(map(norm_s3, s3.glob(bucket.joinpath('*.txt')))) - - -@pytest.mark.parametrize("inp, out", [ - ('/home/user', Path('/home/user')), - (Path('yo'), Path('yo')), - (['hey', 'ho'], [Path('hey'), Path('ho')]), - ({'hey', 'ho'}, {Path('hey'), Path('ho')}), - (('hey', 'ho'), (Path('hey'), Path('ho'))), - (1, None), - (['hey', 1], None), - ('s3://bucket/hey', RemotePath('s3://bucket/hey')), - ({'s3://hey', 's3://ho'}, {RemotePath('s3://hey'), RemotePath('s3://ho')}), -]) + norm_s3 = relative_path_factory("s3://test-bucket") + + with tmpdir.joinpath("test.txt").open(mode="w") as fp: + fp.write("bla") + + assert list(map(norm_local, fs.ls(tmpdir))) == list(map(norm_s3, s3.ls(bucket))) + + assert list(map(norm_local, fs.glob(tmpdir.joinpath("*.txt")))) == list( + map(norm_s3, s3.glob(bucket.joinpath("*.txt"))) + ) + + +@pytest.mark.parametrize( + "inp, out", + [ + ("/home/user", Path("/home/user")), + (Path("yo"), Path("yo")), + (["hey", "ho"], [Path("hey"), Path("ho")]), + ({"hey", "ho"}, {Path("hey"), Path("ho")}), + (("hey", "ho"), (Path("hey"), Path("ho"))), + (1, None), + (["hey", 1], None), + ("s3://bucket/hey", RemotePath("s3://bucket/hey")), + ({"s3://hey", "s3://ho"}, {RemotePath("s3://hey"), RemotePath("s3://ho")}), + ], +) def test_aspath(inp, out): if out is not None: assert out == aspath(inp) @@ -156,44 +158,44 @@ def f(self, s): return s foo = Foo() - assert isinstance(foo.f('hey'), Path) - assert all([isinstance(item, Path) for item in foo.f(['hey', 'ho'])]) + assert isinstance(foo.f("hey"), Path) + assert all([isinstance(item, Path) for item in foo.f(["hey", "ho"])]) def test_memory_fs_rw(): fs = MemoryFileSystem() - with fs.open('memory://some_path/file1.txt', 'wb') as fp: - fp.write(b'hello') + with fs.open("memory://some_path/file1.txt", "wb") as fp: + fp.write(b"hello") - with fs.open('memory://some_path/file2.txt', 'wb') as fp: - fp.write(b'world') + with fs.open("memory://some_path/file2.txt", "wb") as fp: + fp.write(b"world") - assert set(map(str, fs.ls('memory://some_path'))) == { - 'memory://some_path/file1.txt', - 'memory://some_path/file2.txt' + assert set(map(str, fs.ls("memory://some_path"))) == { + "memory://some_path/file1.txt", + "memory://some_path/file2.txt", } - fs.rm('memory://some_path/file1.txt') + fs.rm("memory://some_path/file1.txt") - assert set(map(str, fs.ls('memory://some_path'))) == { - 'memory://some_path/file2.txt' + assert set(map(str, fs.ls("memory://some_path"))) == { + "memory://some_path/file2.txt" } - with fs.open('memory://some_path/file2.txt', 'rb') as fp: + with fs.open("memory://some_path/file2.txt", "rb") as fp: res = fp.read() - assert res == b'world' + assert res == b"world" def test_memory_fs_recursive_rm(): - uri = RemotePath('memory://root/somedir') + uri = RemotePath("memory://root/somedir") fs = MemoryFileSystem() - with fs.open(uri / 'file', 'wb') as fp: - fp.write(b'') - with fs.open(uri / 'subdir' / 'file', 'wb') as fp: - fp.write(b'') + with fs.open(uri / "file", "wb") as fp: + fp.write(b"") + with fs.open(uri / "subdir" / "file", "wb") as fp: + fp.write(b"") fs.rm(uri, recursive=True) @@ -201,26 +203,29 @@ def test_memory_fs_recursive_rm(): def test_list_files(s3_data_dir): - fs = get_fs('s3://s3-test-bucket/', rtype='instance') + fs = get_fs("s3://s3-test-bucket/", rtype="instance") - res = fs.ls('s3://s3-test-bucket/dump/') - assert all([str(p).startswith('s3://s3-') for p in res]) + res = fs.ls("s3://s3-test-bucket/dump/") + assert all([str(p).startswith("s3://s3-") for p in res]) assert len(res) == 10 def test_glob_files(s3_data_dir): - fs = get_fs('s3://s3-test-bucket/', rtype='instance') + fs = get_fs("s3://s3-test-bucket/", rtype="instance") - res = fs.glob('s3://s3-test-bucket/dump/*.csv') - assert all([str(p).startswith('s3://s3-') for p in res]) + res = fs.glob("s3://s3-test-bucket/dump/*.csv") + assert all([str(p).startswith("s3://s3-") for p in res]) assert len(res) == 10 -@pytest.mark.parametrize('scheme, path, exp', [ - ('s3', 's3-bucket/test', 's3://s3-bucket/test'), - ('s3', 's3://s3-bucket/test', 's3://s3-bucket/test'), - ('', '/user/ubuntu/test', 'file://user/ubuntu/test'), - ('', 'file://user/ubuntu/test', 'file://user/ubuntu/test'), -]) +@pytest.mark.parametrize( + "scheme, path, exp", + [ + ("s3", "s3-bucket/test", "s3://s3-bucket/test"), + ("s3", "s3://s3-bucket/test", "s3://s3-bucket/test"), + ("", "/user/ubuntu/test", "file://user/ubuntu/test"), + ("", "file://user/ubuntu/test", "file://user/ubuntu/test"), + ], +) def test_prepend_scheme(scheme, path, exp): assert prepend_scheme(scheme, path) == exp diff --git a/drfs/tests/test_path.py b/drfs/tests/test_path.py index b490659..93fa920 100644 --- a/drfs/tests/test_path.py +++ b/drfs/tests/test_path.py @@ -3,71 +3,60 @@ from drfs import config from drfs.path import DRPath + def test_is_wildcard(): - assert not DRPath('/home').is_wildcard - assert DRPath('/home/*').is_wildcard - assert DRPath('/home/*/yo').is_wildcard - assert DRPath('/home/**/yo').is_wildcard - assert DRPath('/home/*.csv').is_wildcard + assert not DRPath("/home").is_wildcard + assert DRPath("/home/*").is_wildcard + assert DRPath("/home/*/yo").is_wildcard + assert DRPath("/home/**/yo").is_wildcard + assert DRPath("/home/*.csv").is_wildcard - assert not DRPath('s3://bucket').is_wildcard - assert DRPath('s3://bucket/*').is_wildcard - assert DRPath('s3://bucket/*/yo').is_wildcard - assert DRPath('s3://bucket/**/yo').is_wildcard - assert DRPath('s3://bucket/*.csv').is_wildcard + assert not DRPath("s3://bucket").is_wildcard + assert DRPath("s3://bucket/*").is_wildcard + assert DRPath("s3://bucket/*/yo").is_wildcard + assert DRPath("s3://bucket/**/yo").is_wildcard + assert DRPath("s3://bucket/*.csv").is_wildcard def test_is_template(): - assert not DRPath('/home').is_template - assert not DRPath('/home/abc{').is_template - assert not DRPath('/home/abc}{').is_template - assert DRPath('/home/abc{}').is_template - assert DRPath('{}/home/abc').is_template - assert DRPath('/home/abc{yo}').is_template - assert DRPath('/home/abc{yo/100:.2f}').is_template + assert not DRPath("/home").is_template + assert not DRPath("/home/abc{").is_template + assert not DRPath("/home/abc}{").is_template + assert DRPath("/home/abc{}").is_template + assert DRPath("{}/home/abc").is_template + assert DRPath("/home/abc{yo}").is_template + assert DRPath("/home/abc{yo/100:.2f}").is_template def test_remote_div(s3): - p1 = DRPath('s3://test-bucket/') - config["fs_opts"]["s3"] ={"access_key": "test"} + p1 = DRPath("s3://test-bucket/") + config["fs_opts"]["s3"] = {"access_key": "test"} assert p1.storage_options == config["fs_opts"]["s3"].get(dict) config["fs_opts"] = {} - opts = {'key': 'abc', 'secret': 'def'} - p2 = DRPath('s3://test-bucket', storage_options=opts) + opts = {"key": "abc", "secret": "def"} + p2 = DRPath("s3://test-bucket", storage_options=opts) assert p2.storage_options == opts assert p2._acc_real is None p2.exists() # create fs instance with storage options assert p2._acc_real is not None - assert p2._acc_real.fs.key == opts['key'] - assert p2._acc_real.fs.secret == opts['secret'] + assert p2._acc_real.fs.key == opts["key"] + assert p2._acc_real.fs.secret == opts["secret"] - p3 = p2 / 'test.txt' + p3 = p2 / "test.txt" assert p3.storage_options == p2.storage_options assert p3._acc_real is not None assert p3._acc_real is p2._acc_real -@pytest.mark.parametrize( - ("str_path",), - [ - ("s3://test_bucket",), - ("/home/test_dir",), - ] -) +@pytest.mark.parametrize(("str_path",), [("s3://test_bucket",), ("/home/test_dir",)]) def test_path_get_item(str_path): p = DRPath(str_path) assert p[:5] == str_path[:5] -@pytest.mark.parametrize( - ("str_path",), - [ - ("s3://test_bucket",), - ("/home/test_dir",), - ] -) +@pytest.mark.parametrize(("str_path",), [("s3://test_bucket",), ("/home/test_dir",)]) def test_path_startswith(str_path): p = DRPath(str_path) diff --git a/drfs/tests/test_structure.py b/drfs/tests/test_structure.py index 533ecb9..1aa9814 100644 --- a/drfs/tests/test_structure.py +++ b/drfs/tests/test_structure.py @@ -2,14 +2,13 @@ class Structure(Tree): - class data(Tree): - file: P = DRPath('{name}.csv') + file: P = DRPath("{name}.csv") def test_independent_instances(): - structure1 = Structure('/tmp') - structure2 = Structure('/tmp/dir/') + structure1 = Structure("/tmp") + structure2 = Structure("/tmp/dir/") - assert str(structure1.data.file.format(name='file1')) == "/tmp/data/file1.csv" - assert str(structure2.data.file.format(name='file1')) == "/tmp/dir/data/file1.csv" + assert str(structure1.data.file.format(name="file1")) == "/tmp/data/file1.csv" + assert str(structure2.data.file.format(name="file1")) == "/tmp/dir/data/file1.csv" diff --git a/drfs/util.py b/drfs/util.py index 95e4d13..bedd719 100644 --- a/drfs/util.py +++ b/drfs/util.py @@ -16,7 +16,7 @@ def prepend_scheme(scheme, path): """ if scheme == "": scheme = "file" - if path.startswith(scheme + '://'): + if path.startswith(scheme + "://"): return path else: path = path[1:] if path.startswith("/") else path