Cache improvements. (#274)

- Reduce exception clutter from normal cache misses.
- Add settings to fully or selectively disable caching.

Author: JavadocMD

epymorph/cache.py

@@ -3,7 +3,7 @@
 from hashlib import sha256
 from io import BytesIO
 from math import log
-from os import PathLike, getenv
+from os import PathLike
 from pathlib import Path
 from shutil import rmtree
 from sys import modules
@@ -15,28 +15,47 @@
 import requests
 from platformdirs import user_cache_path
 
-
-def _cache_path() -> Path:
-    """
-    Get epymorph's cache directory.
-
-    Returns
-    -------
-    :
-        The path.
-    """
-    if (path_var := getenv("EPYMORPH_CACHE_PATH")) is not None:
-        # Load path from env var
-        path = Path(path_var)
-    else:
-        # fall back to platform-specific default path
-        path = user_cache_path(appname="epymorph")
-    # ensure cache directory exists
-    path.mkdir(parents=True, exist_ok=True)
-    return path
-
-
-CACHE_PATH = _cache_path()
+from epymorph.settings import declare_setting, env_flag, env_path, env_path_list
+
+EPYMORPH_CACHE_PATH = declare_setting(
+    name="EPYMORPH_CACHE_PATH",
+    description=(
+        "Optional path to use as the location to store cached files. "
+        "By default, epymorph uses a path which is appropriate to your OS."
+    ),
+    getter=lambda: env_path(
+        name="EPYMORPH_CACHE_PATH",
+        default_value=user_cache_path(appname="epymorph"),
+        ensure_exists=True,
+    ),
+)
+"""An environment variable for epymorph's cache path."""
+
+EPYMORPH_CACHE_DISABLED = declare_setting(
+    name="EPYMORPH_CACHE_DISABLED",
+    description=(
+        "An optional boolean value; true to disable all cache interactions. "
+        "Default is false."
+    ),
+    getter=lambda: env_flag("EPYMORPH_CACHE_DISABLED", False),
+)
+"""An environment variable to entirely disable caching."""
+
+EPYMORPH_CACHE_DISABLED_PATHS = declare_setting(
+    name="EPYMORPH_CACHE_DISABLED_PATHS",
+    description=(
+        "An optional list of paths (separated by semicolons); "
+        "when attempting to load or save a file using the cache, "
+        "epymorph will check if the cache path starts with one of "
+        "these paths, and if so, interactions with the cache will be "
+        "skipped entirely."
+    ),
+    getter=lambda: env_path_list("EPYMORPH_CACHE_DISABLED_PATHS"),
+)
+"""An environment variable for paths which should have caching disabled."""
+
+
+CACHE_PATH = EPYMORPH_CACHE_PATH.get()
 """The root directory for epymorph's cached files."""
 
 
@@ -398,7 +417,11 @@ def load_file_from_cache(from_path: str | PathLike[str]) -> BytesIO:
     """
     try:
         return load_file(_resolve_cache_path(from_path))
+    except FileMissingError:
+        # missing file is a normal cache miss; no extra context needed
+        raise CacheMissError() from None
     except FileError as e:
+        # any other file error is abnormal and extra context will help debug
         raise CacheMissError() from e
 
 
@@ -425,23 +448,34 @@ def load_or_fetch(cache_path: Path, fetch: Callable[[], BytesIO]) -> BytesIO:
     :
         The file bytes.
     """
-    try:
+    cache_disabled = EPYMORPH_CACHE_DISABLED.get() or any(
+        cache_path.is_relative_to(p)  # is the file's cache path in a disabled path?
+        for p in EPYMORPH_CACHE_DISABLED_PATHS.get()
+    )
+
+    if not cache_disabled:
         # Try to load from cache.
-        return load_file_from_cache(cache_path)
-    except CacheMissError:
-        # On cache miss, fetch file contents.
-        file = fetch()
+        try:
+            return load_file_from_cache(cache_path)
+        except CacheMissError:
+            # passing through the exception context means the cache miss
+            # doesn't clutter up the exception stack if fetching the file
+            # from source fails.
+            pass
+
+    # On cache miss, fetch file contents.
+    file = fetch()
+
+    if not cache_disabled:
         # And attempt to save the file to the cache for next time.
         try:
             save_file_to_cache(cache_path, file)
         except FileWriteError as e:
-            # Failure to save to the cache is not worth stopping the program:
-            # raise a warning.
-            warn(
-                f"Unable to save file to the cache ({cache_path}). Cause:\n{e}",
-                CacheWarning,
-            )
-        return file
+            # Failure to save to the cache is not worth stopping the program.
+            wrn = f"Unable to save file to the cache ({cache_path}). Cause:\n{e}"
+            warn(wrn, CacheWarning)
+
+    return file
 
 
 def load_or_fetch_url(url: str, cache_path: Path) -> BytesIO:

epymorph/geography/us_tiger.py

@@ -278,15 +278,20 @@ def _load_summary_from_cache(
         with np.load(content["data.npz"]) as data_npz:
             return on_hit(**{k: v.tolist() for k, v in data_npz.items()})
     except CacheMissError:
-        data = on_miss()
-        data_bytes = BytesIO()
-        # NOTE: Python doesn't include a type for dataclass instances;
-        # you can import DataclassInstance from _typeshed, but that seems
-        # to break test discovery. Oh well; just ignore this one.
-        model_dict = asdict(data)  # type: ignore
-        np.savez_compressed(data_bytes, **model_dict)
-        save_bundle_to_cache(path, _CACHE_VERSION, {"data.npz": data_bytes})
-        return data
+        # passing through the exception context means the cache miss
+        # doesn't clutter up the exception stack if fetching the file
+        # from source fails.
+        pass
+
+    data = on_miss()
+    data_bytes = BytesIO()
+    # NOTE: Python doesn't include a type for dataclass instances;
+    # you can import DataclassInstance from _typeshed, but that seems
+    # to break test discovery. Oh well; just ignore this one.
+    model_dict = asdict(data)  # type: ignore
+    np.savez_compressed(data_bytes, **model_dict)
+    save_bundle_to_cache(path, _CACHE_VERSION, {"data.npz": data_bytes})
+    return data
 
 
 ##########

epymorph/settings.py

@@ -3,6 +3,7 @@
 """
 
 import os
+from pathlib import Path
 from typing import Callable, Generic, NamedTuple, TypeVar, overload
 from warnings import warn
 
@@ -87,6 +88,91 @@ def env_flag(name: str, default_value: bool | None = None) -> bool | None:
         return default_value
 
 
+@overload
+def env_path(
+    name: str,
+    default_value: Path,
+    *,
+    ensure_exists: bool = False,
+) -> Path: ...
+
+
+@overload
+def env_path(
+    name: str,
+    default_value: None = None,
+    *,
+    ensure_exists: bool = False,
+) -> Path | None: ...
+
+
+def env_path(
+    name: str,
+    default_value: Path | None = None,
+    *,
+    ensure_exists: bool = False,
+) -> Path | None:
+    """
+    Load an environment variable assuming it represents a Path.
+
+    Parameters
+    ----------
+    name :
+        The name of the environment variable to load.
+    default_value :
+        A default value to use in case the variable is not present or can't be
+        interpreted as a Path.
+    ensure_exists :
+        True to attempt to create the Path (as a directory) if it doesn't already exist.
+
+    Returns
+    -------
+    :
+        If the named variable is present and if the value can be interpreted
+        as a Path, return the Path. Else return `default_value`.
+    """
+    value = os.getenv(name)
+    if value is None:
+        path = default_value
+    else:
+        try:
+            path = Path(value)
+        except TypeError:
+            path = default_value
+
+    if path is not None and ensure_exists:
+        path.mkdir(parents=True, exist_ok=True)
+
+    return path
+
+
+def env_path_list(name: str) -> list[Path]:
+    """
+    Load an environment variable assuming it represents a semicolon-separated list of
+    Paths.
+
+    Note: empty string paths will be ignored, since these are likely to be mistakes.
+
+    Parameters
+    ----------
+    name :
+        The name of the environment variable to load.
+
+    Returns
+    -------
+    :
+        If the named variable is present and if the value can be interpreted
+        as a list of Paths, return the Paths. Else return an empty list.
+    """
+    value = os.getenv(name)
+    if value is None:
+        return []
+    try:
+        return [Path(x.strip()) for x in value.split(";") if x.strip() != ""]
+    except TypeError:
+        return []
+
+
 ValueT = TypeVar("ValueT")
 """The type of the value of a setting."""