diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 57d4db26..1a2e16ba 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -2,49 +2,55 @@ name: CI on: [push, pull_request, workflow_dispatch] +defaults: + run: + shell: bash -l -eo pipefail {0} + jobs: test: name: Python ${{ matrix.python-version }} runs-on: ubuntu-latest + timeout-minutes: 10 strategy: fail-fast: false matrix: - python-version: ["3.7", "3.8", "3.9"] + python-version: ["3.7", "3.8", "3.9", "3.10"] steps: - name: Checkout source - uses: actions/checkout@v2 + uses: actions/checkout@v3 - name: Setup Conda Environment - uses: conda-incubator/setup-miniconda@v2 + uses: mamba-org/provision-with-micromamba@main with: - auto-update-conda: true - miniconda-version: latest - activate-environment: test - python-version: ${{ matrix.python-version }} + cache-downloads: true + environment-file: environment_gcsfs.yaml + environment-name: gcsfs_test + extra-specs: | + python=${{ matrix.python-version }} - - name: Install dependencies - shell: bash -l {0} + - name: Conda info run: | - conda install -c conda-forge pytest ujson requests decorator google-auth aiohttp google-auth-oauthlib google-cloud-core google-api-core google-api-python-client -y - pip install git+https://github.com/fsspec/filesystem_spec --no-deps conda list conda --version - - name: Install - shell: bash -l {0} - run: pip install .[crc] + - name: Install libfuse + run: (sudo apt-get install -y fuse || echo "Error installing fuse.") - - name: Run Tests - shell: bash -l {0} + - name: Run tests run: | - export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json - py.test -vv gcsfs + export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json + pytest -vv \ + --log-format="%(asctime)s %(levelname)s %(message)s" \ + --log-date-format="%H:%M:%S" \ + gcsfs/ lint: name: lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v2 - - uses: actions/setup-python@v2 - - uses: pre-commit/action@v2.0.0 + - uses: actions/checkout@v3.1.0 + - uses: actions/setup-python@v4 + with: + python-version: "3.9" + - uses: pre-commit/action@v3.0.0 diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index a7059292..e4f13395 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,16 +1,32 @@ # See https://pre-commit.com for more information # See https://pre-commit.com/hooks.html for more hooks +exclude: versioneer.py repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v3.4.0 + rev: v4.4.0 hooks: - - id: trailing-whitespace - id: end-of-file-fixer - - repo: https://github.com/ambv/black - rev: 22.3.0 + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/psf/black + rev: 22.10.0 hooks: - - id: black - - repo: https://gitlab.com/pycqa/flake8 - rev: 3.8.4 + - id: black + args: + - --target-version=py37 + - repo: https://github.com/pycqa/flake8 + rev: 6.0.0 hooks: - id: flake8 + files: gcsfs/ + - repo: https://github.com/pycqa/isort + rev: 5.10.1 + hooks: + - id: isort + args: ["--profile", "black", "--filter-files"] + - repo: https://github.com/asottile/pyupgrade + rev: v3.2.3 + hooks: + - id: pyupgrade + args: + - --py37-plus diff --git a/LICENSE.txt b/LICENSE.txt index 38b042fe..19079fc5 100644 --- a/LICENSE.txt +++ b/LICENSE.txt @@ -1,4 +1,4 @@ -BSD 3-Clause License +BSD 3-Clause License Copyright (c) 2014-2018, Anaconda, Inc. and contributors All rights reserved. diff --git a/docs/requirements.txt b/docs/requirements.txt index 85674b1c..a7b14f97 100644 --- a/docs/requirements.txt +++ b/docs/requirements.txt @@ -1,2 +1,2 @@ -numpydoc docutils<0.18 +numpydoc diff --git a/docs/source/conf.py b/docs/source/conf.py index 0f6ab1f5..fc8a2d34 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -1,5 +1,4 @@ #!/usr/bin/env python3 -# -*- coding: utf-8 -*- # # GCSFs documentation build configuration file, created by # sphinx-quickstart on Mon Mar 21 15:20:01 2016. @@ -13,8 +12,8 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys import os +import sys # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -118,7 +117,7 @@ # Taken from docs.readthedocs.io: # on_rtd is whether we are on readthedocs.io -on_rtd = os.environ.get("READTHEDOCS", None) == "True" +on_rtd = os.getenv("READTHEDOCS", None) == "True" if not on_rtd: # only import and set the theme if we're building docs locally import sphinx_rtd_theme diff --git a/docs/source/fuse.rst b/docs/source/fuse.rst index 087d4375..8ecc475e 100644 --- a/docs/source/fuse.rst +++ b/docs/source/fuse.rst @@ -23,10 +23,10 @@ In addition to a standard installation of GCSFS, you also need: - fusepy_, which can be installed via conda or pip - pandas, which can also be installed via conda or pip (this library is - used only for its timestring parsing. + used only for its timestring parsing). .. _osxfuse: https://osxfuse.github.io/ -.. _fusepy: https://github.com/terencehonles/fusepy +.. _fusepy: https://github.com/fusepy/fusepy Usage ----- diff --git a/environment_gcsfs.yaml b/environment_gcsfs.yaml new file mode 100644 index 00000000..2d738038 --- /dev/null +++ b/environment_gcsfs.yaml @@ -0,0 +1,21 @@ +name: gcsfs_test +channels: + - conda-forge +dependencies: + - aiohttp + - crcmod + - decorator + - fsspec + - fusepy<3 + - google-api-core + - google-api-python-client + - google-auth + - google-auth-oauthlib + - google-cloud-core + - libfuse<3 + - pytest + - pytest-timeout + - requests + - ujson + - pip: + - git+https://github.com/fsspec/filesystem_spec diff --git a/gcsfs/_version.py b/gcsfs/_version.py index 808e2157..7475623c 100644 --- a/gcsfs/_version.py +++ b/gcsfs/_version.py @@ -84,7 +84,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= stderr=(subprocess.PIPE if hide_stderr else None), ) break - except EnvironmentError: + except OSError: e = sys.exc_info()[1] if e.errno == errno.ENOENT: continue @@ -94,7 +94,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env= return None, None else: if verbose: - print("unable to find command, tried %s" % (commands,)) + print(f"unable to find command, tried {commands}") return None, None stdout = p.communicate()[0].strip() if sys.version_info[0] >= 3: @@ -147,7 +147,7 @@ def git_get_keywords(versionfile_abs): # _version.py. keywords = {} try: - f = open(versionfile_abs, "r") + f = open(versionfile_abs) for line in f.readlines(): if line.strip().startswith("git_refnames ="): mo = re.search(r'=\s*"(.*)"', line) @@ -162,7 +162,7 @@ def git_get_keywords(versionfile_abs): if mo: keywords["date"] = mo.group(1) f.close() - except EnvironmentError: + except OSError: pass return keywords @@ -186,11 +186,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): if verbose: print("keywords are unexpanded, not using") raise NotThisMethod("unexpanded keywords, not a git-archive tarball") - refs = set([r.strip() for r in refnames.strip("()").split(",")]) + refs = {r.strip() for r in refnames.strip("()").split(",")} # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of # just "foo-1.0". If we see a "tag: " prefix, prefer those. TAG = "tag: " - tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)]) + tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)} if not tags: # Either we're using git < 1.8.3, or there really are no tags. We use # a heuristic: assume all version tags have a digit. The old git %d @@ -199,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose): # between branches and tags. By ignoring refnames without digits, we # filter out many common branch names like "release" and # "stabilization", as well as "HEAD" and "". - tags = set([r for r in refs if re.search(r"\d", r)]) + tags = {r for r in refs if re.search(r"\d", r)} if verbose: print("discarding '%s', no digits" % ",".join(refs - tags)) if verbose: @@ -302,7 +302,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command): if verbose: fmt = "tag '%s' doesn't start with prefix '%s'" print(fmt % (full_tag, tag_prefix)) - pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % ( + pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format( full_tag, tag_prefix, ) diff --git a/gcsfs/checkers.py b/gcsfs/checkers.py index 0b6647c2..01091413 100644 --- a/gcsfs/checkers.py +++ b/gcsfs/checkers.py @@ -1,9 +1,9 @@ -from base64 import b64encode import base64 -from typing import Optional +from base64 import b64encode from hashlib import md5 -from .retry import ChecksumError +from typing import Optional +from .retry import ChecksumError try: import crcmod diff --git a/gcsfs/cli/gcsfuse.py b/gcsfs/cli/gcsfuse.py index a64c420c..a23f09dd 100644 --- a/gcsfs/cli/gcsfuse.py +++ b/gcsfs/cli/gcsfuse.py @@ -1,5 +1,6 @@ -import click import logging + +import click from fuse import FUSE from gcsfs.gcsfuse import GCSFS @@ -54,7 +55,7 @@ def main( if verbose > 1: logging.basicConfig(level=logging.DEBUG, format=fmt) - print("Mounting bucket %s to directory %s" % (bucket, mount_point)) + print(f"Mounting bucket {bucket} to directory {mount_point}") print("foreground:", foreground, ", nothreads:", not threads) FUSE( GCSFS(bucket, token=token, project=project_id, nfiles=cache_files), diff --git a/gcsfs/core.py b/gcsfs/core.py index e4fc4226..6f2e6933 100644 --- a/gcsfs/core.py +++ b/gcsfs/core.py @@ -1,10 +1,7 @@ -# -*- coding: utf-8 -*- """ Google Cloud Storage pythonic interface """ import asyncio -import fsspec - import io import json import logging @@ -13,23 +10,26 @@ import re import warnings import weakref +from urllib.parse import parse_qs +from urllib.parse import quote as quote_urllib +from urllib.parse import urlsplit -from fsspec.asyn import sync_wrapper, sync, AsyncFileSystem -from fsspec.utils import stringify_path, setup_logging +import fsspec +from fsspec.asyn import AsyncFileSystem, sync, sync_wrapper from fsspec.callbacks import NoOpCallback from fsspec.implementations.http import get_client -from .retry import retry_request, validate_response +from fsspec.utils import setup_logging, stringify_path + +from . import __version__ as version from .checkers import get_consistency_checker from .credentials import GoogleCredentials -from . import __version__ as version -from urllib.parse import quote as quote_urllib -from urllib.parse import parse_qs, urlsplit +from .retry import retry_request, validate_response logger = logging.getLogger("gcsfs") if "GCSFS_DEBUG" in os.environ: - setup_logging(logger=logger, level=os.environ["GCSFS_DEBUG"]) + setup_logging(logger=logger, level=os.getenv("GCSFS_DEBUG")) # client created 2018-01-16 @@ -48,7 +48,7 @@ "publicRead", "publicReadWrite", } -DEFAULT_PROJECT = os.environ.get("GCSFS_DEFAULT_PROJECT", "") +DEFAULT_PROJECT = os.getenv("GCSFS_DEFAULT_PROJECT", "") GCS_MIN_BLOCK_SIZE = 2**18 GCS_MAX_BLOCK_SIZE = 2**28 @@ -105,7 +105,7 @@ def _location(): ------- valid http location """ - _emulator_location = os.environ.get("STORAGE_EMULATOR_HOST", None) + _emulator_location = os.getenv("STORAGE_EMULATOR_HOST", None) return ( _emulator_location if _emulator_location else "https://storage.googleapis.com" ) @@ -850,7 +850,7 @@ def url(self, path): self._location, bucket, object, - "&generation={}".format(generation) if generation else "", + f"&generation={generation}" if generation else "", ) async def _cat_file(self, path, start=None, end=None, **kwargs): @@ -1176,7 +1176,7 @@ async def _put_file( async def _isdir(self, path): try: return (await self._info(path))["type"] == "directory" - except IOError: + except OSError: return False async def _find( @@ -1731,11 +1731,7 @@ async def simple_upload( template = ( "--==0==" "\nContent-Type: application/json; charset=UTF-8" - "\n\n" - + metadata - + "\n--==0==" - + "\nContent-Type: {0}".format(content_type) - + "\n\n" + "\n\n" + metadata + "\n--==0==" + f"\nContent-Type: {content_type}" + "\n\n" ) data = template.encode() + datain + b"\n--==0==--" diff --git a/gcsfs/credentials.py b/gcsfs/credentials.py index ffe9cd00..05439b93 100644 --- a/gcsfs/credentials.py +++ b/gcsfs/credentials.py @@ -1,21 +1,20 @@ +import json +import logging +import os +import pickle import textwrap +import threading +import warnings import google.auth as gauth import google.auth.compute_engine import google.auth.credentials import google.auth.exceptions +import requests +from google.auth.transport.requests import Request +from google.oauth2 import service_account from google.oauth2.credentials import Credentials from google_auth_oauthlib.flow import InstalledAppFlow -from google.oauth2 import service_account -from google.auth.transport.requests import Request -import json -import requests -import os -import pickle -import requests -import threading -import warnings -import logging logger = logging.getLogger("gcsfs.credentials") @@ -154,7 +153,8 @@ def _connect_token(self, token): # TODO: catch specific exceptions # some other kind of token file # will raise exception if is not json - token = json.load(open(token)) + with open(token) as data: + token = json.load(data) if isinstance(token, dict): credentials = self._dict_to_credentials(token) elif isinstance(token, google.auth.credentials.Credentials): diff --git a/gcsfs/retry.py b/gcsfs/retry.py index b8b8b891..967ad691 100644 --- a/gcsfs/retry.py +++ b/gcsfs/retry.py @@ -1,13 +1,12 @@ import asyncio -from decorator import decorator import json import logging import random - -import requests.exceptions -import google.auth.exceptions import aiohttp.client_exceptions +import google.auth.exceptions +import requests.exceptions +from decorator import decorator logger = logging.getLogger("gcsfs") @@ -28,7 +27,7 @@ def __init__(self, error_response=None): self.message = "" self.code = None # Call the base class constructor with the parameters it needs - super(HttpError, self).__init__(self.message) + super().__init__(self.message) class ChecksumError(Exception): @@ -93,11 +92,11 @@ def validate_response(status, content, path, args=None): msg = content if status == 403: - raise IOError("Forbidden: %s\n%s" % (path, msg)) + raise OSError(f"Forbidden: {path}\n{msg}") elif status == 502: raise requests.exceptions.ProxyError() elif "invalid" in str(msg): - raise ValueError("Bad Request: %s\n%s" % (path, msg)) + raise ValueError(f"Bad Request: {path}\n{msg}") elif error: raise HttpError(error) elif status: @@ -141,12 +140,10 @@ async def retry_request(func, retries=6, *args, **kwargs): logger.debug("Request returned 404, no retries.") raise e if retry == retries - 1: - logger.exception( - "%s out of retries on exception: %s" % (func.__name__, e) - ) + logger.exception(f"{func.__name__} out of retries on exception: {e}") raise e if is_retriable(e): - logger.debug("%s retrying after exception: %s" % (func.__name__, e)) + logger.debug(f"{func.__name__} retrying after exception: {e}") continue - logger.exception("%s non-retriable exception: %s" % (func.__name__, e)) + logger.exception(f"{func.__name__} non-retriable exception: {e}") raise e diff --git a/gcsfs/tests/conftest.py b/gcsfs/tests/conftest.py index 893dbedd..b00fbb61 100644 --- a/gcsfs/tests/conftest.py +++ b/gcsfs/tests/conftest.py @@ -58,7 +58,7 @@ def stop_docker(container): def docker_gcs(): if "STORAGE_EMULATOR_HOST" in os.environ: # assume using real API or otherwise have a server already set up - yield os.environ["STORAGE_EMULATOR_HOST"] + yield os.getenv("STORAGE_EMULATOR_HOST") return container = "gcsfs_test" cmd = ( diff --git a/gcsfs/tests/settings.py b/gcsfs/tests/settings.py index e535d925..0e074f97 100644 --- a/gcsfs/tests/settings.py +++ b/gcsfs/tests/settings.py @@ -1,5 +1,5 @@ import os -TEST_BUCKET = os.environ.get("GCSFS_TEST_BUCKET", "gcsfs_test") -TEST_PROJECT = os.environ.get("GCSFS_TEST_PROJECT", "project") +TEST_BUCKET = os.getenv("GCSFS_TEST_BUCKET", "gcsfs_test") +TEST_PROJECT = os.getenv("GCSFS_TEST_PROJECT", "project") TEST_REQUESTER_PAYS_BUCKET = "gcsfs_test_req_pay" diff --git a/gcsfs/tests/test_checkers.py b/gcsfs/tests/test_checkers.py index 90319fa8..47d74226 100644 --- a/gcsfs/tests/test_checkers.py +++ b/gcsfs/tests/test_checkers.py @@ -1,10 +1,11 @@ -from gcsfs.retry import ChecksumError -from gcsfs.checkers import Crc32cChecker, MD5Checker, SizeChecker, crcmod -from hashlib import md5 import base64 +from hashlib import md5 import pytest +from gcsfs.checkers import Crc32cChecker, MD5Checker, SizeChecker, crcmod +from gcsfs.retry import ChecksumError + def google_response_from_data(expected_data: bytes, actual_data=None): diff --git a/gcsfs/tests/test_core.py b/gcsfs/tests/test_core.py index aa395f68..05a607e7 100644 --- a/gcsfs/tests/test_core.py +++ b/gcsfs/tests/test_core.py @@ -1,32 +1,22 @@ -# -*- coding: utf-8 -*- - import io from builtins import FileNotFoundError from itertools import chain from unittest import mock -from urllib.parse import urlparse, parse_qs, unquote +from urllib.parse import parse_qs, unquote, urlparse from uuid import uuid4 import pytest import requests - -from fsspec.utils import seek_delimiter from fsspec.asyn import sync +from fsspec.utils import seek_delimiter -from gcsfs.tests.settings import TEST_BUCKET, TEST_PROJECT, TEST_REQUESTER_PAYS_BUCKET -from gcsfs.tests.conftest import ( - a, - allfiles, - b, - csv_files, - files, - text_files, -) -from gcsfs.tests.utils import tempdir, tmpfile -from gcsfs.core import GCSFileSystem, quote -from gcsfs.credentials import GoogleCredentials import gcsfs.checkers from gcsfs import __version__ as version +from gcsfs.core import GCSFileSystem, quote +from gcsfs.credentials import GoogleCredentials +from gcsfs.tests.conftest import a, allfiles, b, csv_files, files, text_files +from gcsfs.tests.settings import TEST_BUCKET, TEST_PROJECT, TEST_REQUESTER_PAYS_BUCKET +from gcsfs.tests.utils import tempdir, tmpfile def test_simple(gcs): @@ -167,10 +157,10 @@ def test_ls_touch(gcs): gcs.touch(b) L = gcs.ls(TEST_BUCKET + "/tmp/test", False) - assert set(L) == set([a, b]) + assert set(L) == {a, b} L_d = gcs.ls(TEST_BUCKET + "/tmp/test", True) - assert set(d["name"] for d in L_d) == set([a, b]) + assert {d["name"] for d in L_d} == {a, b} def test_rm(gcs): @@ -1077,7 +1067,7 @@ def test_percent_file_name(gcs): fn2 = unquote(fn) gcs.touch(fn2) assert gcs.cat(fn2) != data - assert set(gcs.ls(parent)) == set([fn, fn2]) + assert set(gcs.ls(parent)) == {fn, fn2} @pytest.mark.parametrize( @@ -1203,10 +1193,11 @@ def test_ls_versioned(gcs_versioned): wo.write(b"v2") v2 = gcs_versioned.info(a)["generation"] dpath = posixpath.dirname(a) - assert {f"{a}#{v1}", f"{a}#{v2}"} == set(gcs_versioned.ls(dpath, versions=True)) - assert {f"{a}#{v1}", f"{a}#{v2}"} == set( + versions = {f"{a}#{v1}", f"{a}#{v2}"} + assert versions == set(gcs_versioned.ls(dpath, versions=True)) + assert versions == { entry["name"] for entry in gcs_versioned.ls(dpath, detail=True, versions=True) - ) + } def test_find_versioned(gcs_versioned): @@ -1216,7 +1207,6 @@ def test_find_versioned(gcs_versioned): with gcs_versioned.open(a, "wb") as wo: wo.write(b"v2") v2 = gcs_versioned.info(a)["generation"] - assert {f"{a}#{v1}", f"{a}#{v2}"} == set(gcs_versioned.find(a, versions=True)) - assert {f"{a}#{v1}", f"{a}#{v2}"} == set( - gcs_versioned.find(a, detail=True, versions=True) - ) + versions = {f"{a}#{v1}", f"{a}#{v2}"} + assert versions == set(gcs_versioned.find(a, versions=True)) + assert versions == set(gcs_versioned.find(a, detail=True, versions=True)) diff --git a/gcsfs/tests/test_fuse.py b/gcsfs/tests/test_fuse.py index e7cea49e..f1f494cd 100644 --- a/gcsfs/tests/test_fuse.py +++ b/gcsfs/tests/test_fuse.py @@ -1,39 +1,63 @@ +import logging import os +import sys +import tempfile +import threading +import time +from functools import partial import pytest -import tempfile - -fuse = pytest.importorskip("fuse") -from fsspec.fuse import run from gcsfs.tests.settings import TEST_BUCKET -import threading -import time -def test_fuse(gcs): +@pytest.mark.timeout(180) +@pytest.fixture +def fsspec_fuse_run(): + """Fixture catches other errors on fuse import.""" + try: + _fuse = pytest.importorskip("fuse") # noqa + + from fsspec.fuse import run as _fsspec_fuse_run + + return _fsspec_fuse_run + except Exception as error: + logging.debug("Error importing fuse: %s", error) + pytest.skip("Error importing fuse.") + + +@pytest.mark.skipif(sys.version_info < (3, 9), reason="Test fuse causes hang.") +@pytest.mark.xfail(reason="Failing test not previously tested.") +@pytest.mark.timeout(180) +def test_fuse(gcs, fsspec_fuse_run): mountpath = tempfile.mkdtemp() - th = threading.Thread(target=lambda: run(gcs, TEST_BUCKET + "/", mountpath)) + _run = partial(fsspec_fuse_run, gcs, TEST_BUCKET + "/", mountpath) + th = threading.Thread(target=_run) th.daemon = True th.start() time.sleep(5) timeout = 20 - while True: + n = 40 + for i in range(n): + logging.debug(f"Attempt # {i+1}/{n} to create lock file.") try: open(os.path.join(mountpath, "lock"), "w").close() os.remove(os.path.join(mountpath, "lock")) break - except: # noqa: E722 + except Exception as error: # noqa: E722 + logging.debug("Error: %s", error) time.sleep(0.5) timeout -= 0.5 assert timeout > 0 + else: + raise AssertionError(f"Attempted lock file failed after {n} attempts.") with open(os.path.join(mountpath, "hello"), "w") as f: # NB this is in TEXT mode f.write("hello") files = os.listdir(mountpath) assert "hello" in files - with open(os.path.join(mountpath, "hello"), "r") as f: + with open(os.path.join(mountpath, "hello")) as f: # NB this is in TEXT mode assert f.read() == "hello" diff --git a/gcsfs/tests/test_manyopens.py b/gcsfs/tests/test_manyopens.py index af7bb1de..9c46c190 100644 --- a/gcsfs/tests/test_manyopens.py +++ b/gcsfs/tests/test_manyopens.py @@ -1,4 +1,3 @@ -# -*- coding: utf-8 -*- """ Test helper to open the same file many times. @@ -9,8 +8,9 @@ Ideally you should see nothing, just the attempt count go up until we're done. """ -from __future__ import print_function + import sys + import gcsfs diff --git a/gcsfs/tests/test_mapping.py b/gcsfs/tests/test_mapping.py index 70788ead..b8162da2 100644 --- a/gcsfs/tests/test_mapping.py +++ b/gcsfs/tests/test_mapping.py @@ -1,7 +1,8 @@ import pytest + from gcsfs.tests.settings import TEST_BUCKET -root = TEST_BUCKET + "/mapping" +MAPPING_ROOT = TEST_BUCKET + "/mapping" def test_api(): @@ -12,7 +13,7 @@ def test_api(): def test_map_simple(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) assert not d assert list(d) == list(d.keys()) == [] @@ -21,12 +22,12 @@ def test_map_simple(gcs): def test_map_default_gcsfilesystem(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) assert d.fs is gcs def test_map_errors(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) with pytest.raises(KeyError): d["nonexistent"] try: @@ -36,7 +37,7 @@ def test_map_errors(gcs): def test_map_with_data(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = b"123" assert list(d) == list(d.keys()) == ["x"] assert list(d.values()) == [b"123"] @@ -44,7 +45,7 @@ def test_map_with_data(gcs): assert d["x"] == b"123" assert bool(d) - assert gcs.find(root) == [TEST_BUCKET + "/mapping/x"] + assert gcs.find(MAPPING_ROOT) == [TEST_BUCKET + "/mapping/x"] d["x"] = b"000" assert d["x"] == b"000" @@ -57,7 +58,7 @@ def test_map_with_data(gcs): def test_map_complex_keys(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d[1] = b"hello" assert d[1] == b"hello" del d[1] @@ -73,7 +74,7 @@ def test_map_complex_keys(gcs): def test_map_clear_empty(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d.clear() assert list(d) == [] d[1] = b"1" @@ -84,7 +85,7 @@ def test_map_clear_empty(gcs): def test_map_pickle(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = b"1" assert d["x"] == b"1" @@ -98,14 +99,14 @@ def test_map_pickle(gcs): def test_map_array(gcs): from array import array - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = array("B", [65] * 1000) assert d["x"] == b"A" * 1000 def test_map_bytearray(gcs): - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = bytearray(b"123") assert d["x"] == b"123" @@ -134,7 +135,7 @@ def test_new_bucket(gcs): def test_map_pickle(gcs): import pickle - d = gcs.get_mapper(root) + d = gcs.get_mapper(MAPPING_ROOT) d["x"] = b"1234567890" b = pickle.dumps(d) diff --git a/gcsfs/tests/test_retry.py b/gcsfs/tests/test_retry.py index e452b667..306c99af 100644 --- a/gcsfs/tests/test_retry.py +++ b/gcsfs/tests/test_retry.py @@ -1,10 +1,11 @@ import os + +import pytest import requests from requests.exceptions import ProxyError -import pytest -from gcsfs.tests.settings import TEST_BUCKET from gcsfs.retry import HttpError, is_retriable, validate_response +from gcsfs.tests.settings import TEST_BUCKET from gcsfs.tests.utils import tmpfile diff --git a/gcsfs/tests/utils.py b/gcsfs/tests/utils.py index a07c59f6..720d08fa 100644 --- a/gcsfs/tests/utils.py +++ b/gcsfs/tests/utils.py @@ -1,7 +1,7 @@ -from contextlib import contextmanager import os import shutil import tempfile +from contextlib import contextmanager @contextmanager diff --git a/requirements.txt b/requirements.txt index 9f815cbb..5f87b743 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ +aiohttp!=4.0.0a0, !=4.0.0a1 +decorator>4.1.2 +fsspec==2022.11.0 google-auth>=1.2 google-auth-oauthlib google-cloud-storage requests -decorator>4.1.2 -fsspec==2022.11.0 -aiohttp!=4.0.0a0, !=4.0.0a1 diff --git a/setup.cfg b/setup.cfg index fdfaee20..fda24791 100644 --- a/setup.cfg +++ b/setup.cfg @@ -11,18 +11,30 @@ universal=1 [flake8] exclude = versioneer.py,docs/source/conf.py ignore = - E20, # Extra space in brackets - E231,E241, # Multiple spaces around "," - E26, # Comments - E4, # Import formatting - E721, # Comparing types instead of isinstance - E731, # Assigning lambda expression - E741, # Ambiguous variable names - W503, # line break before binary operator - W504, # line break after binary operator - F811, # redefinition of unused 'loop' from line 10 + # Extra space in brackets + E20, + # Multiple spaces around "," + E231,E241, + # Comments + E26, + # Import formatting + E4, + # Comparing types instead of isinstance + E721, + # Assigning lambda expression + E731, + # Ambiguous variable names + E741, + # line break before binary operator + W503, + # line break after binary operator + W504, + # redefinition of unused 'loop' from line 10 + F811, max-line-length = 120 [tool:pytest] addopts = - --color=yes + --color=yes --timeout=600 +log_cli = false +log_cli_level = DEBUG diff --git a/setup.py b/setup.py index 8dc2aa3d..90712349 100755 --- a/setup.py +++ b/setup.py @@ -1,9 +1,10 @@ #!/usr/bin/env python import os + from setuptools import setup -import versioneer +import versioneer setup( name="gcsfs", @@ -22,6 +23,7 @@ "Programming Language :: Python :: 3.7", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], keywords=["google-cloud-storage", "gcloud", "file-system"], packages=["gcsfs", "gcsfs.cli"],