Update pre-commit, add python versions (#508)

* Update pre-commit, add python versions

* Skip fuse test for some python versions

.github/workflows/ci.yml

@@ -2,49 +2,55 @@ name: CI
 
 on: [push, pull_request, workflow_dispatch]
 
+defaults:
+  run:
+    shell: bash -l -eo pipefail {0}
+
 jobs:
   test:
     name: Python ${{ matrix.python-version }}
     runs-on: ubuntu-latest
+    timeout-minutes: 10
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.7", "3.8", "3.9"]
+        python-version: ["3.7", "3.8", "3.9", "3.10"]
 
     steps:
       - name: Checkout source
-        uses: actions/checkout@v2
+        uses: actions/checkout@v3
 
       - name: Setup Conda Environment
-        uses: conda-incubator/setup-miniconda@v2
+        uses: mamba-org/provision-with-micromamba@main
         with:
-          auto-update-conda: true
-          miniconda-version: latest
-          activate-environment: test
-          python-version: ${{ matrix.python-version }}
+          cache-downloads: true
+          environment-file: environment_gcsfs.yaml
+          environment-name: gcsfs_test
+          extra-specs: |
+            python=${{ matrix.python-version }}
 
-      - name: Install dependencies
-        shell: bash -l {0}
+      - name: Conda info
         run: |
-          conda install -c conda-forge pytest ujson requests decorator google-auth aiohttp google-auth-oauthlib google-cloud-core google-api-core google-api-python-client -y
-          pip install git+https://github.com/fsspec/filesystem_spec --no-deps
           conda list
           conda --version
 
-      - name: Install
-        shell: bash -l {0}
-        run: pip install .[crc]
+      - name: Install libfuse
+        run: (sudo apt-get install -y fuse || echo "Error installing fuse.")
 
-      - name: Run Tests
-        shell: bash -l {0}
+      - name: Run tests
         run: |
-            export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json
-            py.test -vv gcsfs
+          export GOOGLE_APPLICATION_CREDENTIALS=$(pwd)/gcsfs/tests/fake-secret.json
+          pytest -vv \
+          --log-format="%(asctime)s %(levelname)s %(message)s" \
+          --log-date-format="%H:%M:%S" \
+          gcsfs/
 
   lint:
     name: lint
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
-    - uses: actions/setup-python@v2
-    - uses: pre-commit/[email protected]
+      - uses: actions/[email protected]
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.9"
+      - uses: pre-commit/[email protected]

.pre-commit-config.yaml

@@ -1,16 +1,32 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
+exclude: versioneer.py
 repos:
   - repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v3.4.0
+    rev: v4.4.0
     hooks:
-      - id: trailing-whitespace
       - id: end-of-file-fixer
-  - repo: https://github.com/ambv/black
-    rev: 22.3.0
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/psf/black
+    rev: 22.10.0
     hooks:
-    - id: black
-  - repo: https://gitlab.com/pycqa/flake8
-    rev: 3.8.4
+      - id: black
+        args:
+          - --target-version=py37
+  - repo: https://github.com/pycqa/flake8
+    rev: 6.0.0
     hooks:
       - id: flake8
+        files: gcsfs/
+  - repo: https://github.com/pycqa/isort
+    rev: 5.10.1
+    hooks:
+      - id: isort
+        args: ["--profile", "black", "--filter-files"]
+  - repo: https://github.com/asottile/pyupgrade
+    rev: v3.2.3
+    hooks:
+      - id: pyupgrade
+        args:
+          - --py37-plus

LICENSE.txt

@@ -1,4 +1,4 @@
-BSD 3-Clause License
+BSD 3-Clause License
 
 Copyright (c) 2014-2018, Anaconda, Inc. and contributors
 All rights reserved.

docs/requirements.txt

@@ -1,2 +1,2 @@
-numpydoc
 docutils<0.18
+numpydoc

docs/source/conf.py

@@ -1,5 +1,4 @@
 #!/usr/bin/env python3
-# -*- coding: utf-8 -*-
 #
 # GCSFs documentation build configuration file, created by
 # sphinx-quickstart on Mon Mar 21 15:20:01 2016.
@@ -13,8 +12,8 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys
 import os
+import sys
 
 # If extensions (or modules to document with autodoc) are in another directory,
 # add these directories to sys.path here. If the directory is relative to the
@@ -118,7 +117,7 @@
 
 # Taken from docs.readthedocs.io:
 # on_rtd is whether we are on readthedocs.io
-on_rtd = os.environ.get("READTHEDOCS", None) == "True"
+on_rtd = os.getenv("READTHEDOCS", None) == "True"
 
 if not on_rtd:  # only import and set the theme if we're building docs locally
     import sphinx_rtd_theme

docs/source/fuse.rst

@@ -23,10 +23,10 @@ In addition to a standard installation of GCSFS, you also need:
    - fusepy_, which can be installed via conda or pip
 
    - pandas, which can also be installed via conda or pip (this library is
-     used only for its timestring parsing.
+     used only for its timestring parsing).
 
 .. _osxfuse: https://osxfuse.github.io/
-.. _fusepy: https://github.com/terencehonles/fusepy
+.. _fusepy: https://github.com/fusepy/fusepy
 
 Usage
 -----

environment_gcsfs.yaml

@@ -0,0 +1,21 @@
+name: gcsfs_test
+channels:
+  - conda-forge
+dependencies:
+  - aiohttp
+  - crcmod
+  - decorator
+  - fsspec
+  - fusepy<3
+  - google-api-core
+  - google-api-python-client
+  - google-auth
+  - google-auth-oauthlib
+  - google-cloud-core
+  - libfuse<3
+  - pytest
+  - pytest-timeout
+  - requests
+  - ujson
+  - pip:
+      - git+https://github.com/fsspec/filesystem_spec

gcsfs/_version.py

@@ -84,7 +84,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
                 stderr=(subprocess.PIPE if hide_stderr else None),
             )
             break
-        except EnvironmentError:
+        except OSError:
             e = sys.exc_info()[1]
             if e.errno == errno.ENOENT:
                 continue
@@ -94,7 +94,7 @@ def run_command(commands, args, cwd=None, verbose=False, hide_stderr=False, env=
             return None, None
     else:
         if verbose:
-            print("unable to find command, tried %s" % (commands,))
+            print(f"unable to find command, tried {commands}")
         return None, None
     stdout = p.communicate()[0].strip()
     if sys.version_info[0] >= 3:
@@ -147,7 +147,7 @@ def git_get_keywords(versionfile_abs):
     # _version.py.
     keywords = {}
     try:
-        f = open(versionfile_abs, "r")
+        f = open(versionfile_abs)
         for line in f.readlines():
             if line.strip().startswith("git_refnames ="):
                 mo = re.search(r'=\s*"(.*)"', line)
@@ -162,7 +162,7 @@ def git_get_keywords(versionfile_abs):
                 if mo:
                     keywords["date"] = mo.group(1)
         f.close()
-    except EnvironmentError:
+    except OSError:
         pass
     return keywords
 
@@ -186,11 +186,11 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         if verbose:
             print("keywords are unexpanded, not using")
         raise NotThisMethod("unexpanded keywords, not a git-archive tarball")
-    refs = set([r.strip() for r in refnames.strip("()").split(",")])
+    refs = {r.strip() for r in refnames.strip("()").split(",")}
     # starting in git-1.8.3, tags are listed as "tag: foo-1.0" instead of
     # just "foo-1.0". If we see a "tag: " prefix, prefer those.
     TAG = "tag: "
-    tags = set([r[len(TAG) :] for r in refs if r.startswith(TAG)])
+    tags = {r[len(TAG) :] for r in refs if r.startswith(TAG)}
     if not tags:
         # Either we're using git < 1.8.3, or there really are no tags. We use
         # a heuristic: assume all version tags have a digit. The old git %d
@@ -199,7 +199,7 @@ def git_versions_from_keywords(keywords, tag_prefix, verbose):
         # between branches and tags. By ignoring refnames without digits, we
         # filter out many common branch names like "release" and
         # "stabilization", as well as "HEAD" and "".
-        tags = set([r for r in refs if re.search(r"\d", r)])
+        tags = {r for r in refs if re.search(r"\d", r)}
         if verbose:
             print("discarding '%s', no digits" % ",".join(refs - tags))
     if verbose:
@@ -302,7 +302,7 @@ def git_pieces_from_vcs(tag_prefix, root, verbose, run_command=run_command):
             if verbose:
                 fmt = "tag '%s' doesn't start with prefix '%s'"
                 print(fmt % (full_tag, tag_prefix))
-            pieces["error"] = "tag '%s' doesn't start with prefix '%s'" % (
+            pieces["error"] = "tag '{}' doesn't start with prefix '{}'".format(
                 full_tag,
                 tag_prefix,
             )

gcsfs/checkers.py

@@ -1,9 +1,9 @@
-from base64 import b64encode
 import base64
-from typing import Optional
+from base64 import b64encode
 from hashlib import md5
-from .retry import ChecksumError
+from typing import Optional
 
+from .retry import ChecksumError
 
 try:
     import crcmod

gcsfs/cli/gcsfuse.py

@@ -1,5 +1,6 @@
-import click
 import logging
+
+import click
 from fuse import FUSE
 
 from gcsfs.gcsfuse import GCSFS
@@ -54,7 +55,7 @@ def main(
     if verbose > 1:
         logging.basicConfig(level=logging.DEBUG, format=fmt)
 
-    print("Mounting bucket %s to directory %s" % (bucket, mount_point))
+    print(f"Mounting bucket {bucket} to directory {mount_point}")
     print("foreground:", foreground, ", nothreads:", not threads)
     FUSE(
         GCSFS(bucket, token=token, project=project_id, nfiles=cache_files),

gcsfs/core.py

@@ -1,10 +1,7 @@
-# -*- coding: utf-8 -*-
 """
 Google Cloud Storage pythonic interface
 """
 import asyncio
-import fsspec
-
 import io
 import json
 import logging
@@ -13,23 +10,26 @@
 import re
 import warnings
 import weakref
+from urllib.parse import parse_qs
+from urllib.parse import quote as quote_urllib
+from urllib.parse import urlsplit
 
-from fsspec.asyn import sync_wrapper, sync, AsyncFileSystem
-from fsspec.utils import stringify_path, setup_logging
+import fsspec
+from fsspec.asyn import AsyncFileSystem, sync, sync_wrapper
 from fsspec.callbacks import NoOpCallback
 from fsspec.implementations.http import get_client
-from .retry import retry_request, validate_response
+from fsspec.utils import setup_logging, stringify_path
+
+from . import __version__ as version
 from .checkers import get_consistency_checker
 from .credentials import GoogleCredentials
-from . import __version__ as version
-from urllib.parse import quote as quote_urllib
-from urllib.parse import parse_qs, urlsplit
+from .retry import retry_request, validate_response
 
 logger = logging.getLogger("gcsfs")
 
 
 if "GCSFS_DEBUG" in os.environ:
-    setup_logging(logger=logger, level=os.environ["GCSFS_DEBUG"])
+    setup_logging(logger=logger, level=os.getenv("GCSFS_DEBUG"))
 
 
 # client created 2018-01-16
@@ -48,7 +48,7 @@
     "publicRead",
     "publicReadWrite",
 }
-DEFAULT_PROJECT = os.environ.get("GCSFS_DEFAULT_PROJECT", "")
+DEFAULT_PROJECT = os.getenv("GCSFS_DEFAULT_PROJECT", "")
 
 GCS_MIN_BLOCK_SIZE = 2**18
 GCS_MAX_BLOCK_SIZE = 2**28
@@ -105,7 +105,7 @@ def _location():
     -------
     valid http location
     """
-    _emulator_location = os.environ.get("STORAGE_EMULATOR_HOST", None)
+    _emulator_location = os.getenv("STORAGE_EMULATOR_HOST", None)
     return (
         _emulator_location if _emulator_location else "https://storage.googleapis.com"
     )
@@ -850,7 +850,7 @@ def url(self, path):
             self._location,
             bucket,
             object,
-            "&generation={}".format(generation) if generation else "",
+            f"&generation={generation}" if generation else "",
         )
 
     async def _cat_file(self, path, start=None, end=None, **kwargs):
@@ -1176,7 +1176,7 @@ async def _put_file(
     async def _isdir(self, path):
         try:
             return (await self._info(path))["type"] == "directory"
-        except IOError:
+        except OSError:
             return False
 
     async def _find(
@@ -1731,11 +1731,7 @@ async def simple_upload(
     template = (
         "--==0=="
         "\nContent-Type: application/json; charset=UTF-8"
-        "\n\n"
-        + metadata
-        + "\n--==0=="
-        + "\nContent-Type: {0}".format(content_type)
-        + "\n\n"
+        "\n\n" + metadata + "\n--==0==" + f"\nContent-Type: {content_type}" + "\n\n"
     )
 
     data = template.encode() + datain + b"\n--==0==--"