tests: Skip snapshot tests if Textual is outdated

Have our test suite check the version of Textual and
pytest-textual-snapshot that it's running with, and automatically
disable the Textual snapshot tests if we can determine statically that
they will go on to fail. This is a better experience for 3rd party
package maintainers and simplifies our handling of Python versions that
Textual has already dropped support for. This doesn't negatively affect
our CI, since we keep our dependencies live at head in CI.

Signed-off-by: Matt Wozniski <[email protected]>

requirements-test.txt

@@ -8,3 +8,4 @@ setuptools; python_version >= '3.12'
 pkgconfig
 pytest-textual-snapshot
 textual >= 0.43, != 0.65.2, != 0.66
+packaging

setup.py

@@ -120,6 +120,7 @@ def build_js_files(self):
     "setuptools; python_version >= '3.12'",
     "pytest-textual-snapshot",
     "textual >= 0.43, != 0.65.2, != 0.66",
+    "packaging",
 ]
 
 benchmark_requires = [

tests/conftest.py

@@ -2,6 +2,12 @@
 import sys
 
 import pytest
+from packaging import version
+
+SNAPSHOT_MINIMUM_VERSIONS = {
+    "textual": "0.73",
+    "pytest-textual-snapshot": "1.0",
+}
 
 
 @pytest.fixture
@@ -13,7 +19,48 @@ def free_port():
     return port_number
 
 
-if sys.version_info < (3, 8):
-    # Ignore unused Textual snapshots on Python 3.7
-    def pytest_configure(config):
+def _snapshot_skip_reason():
+    if sys.version_info < (3, 8):
+        # Every version available for 3.7 is too old
+        return f"snapshot tests require textual>={SNAPSHOT_MINIMUM_VERSIONS['textual']}"
+
+    from importlib import metadata  # Added in 3.8
+
+    for lib, min_ver in SNAPSHOT_MINIMUM_VERSIONS.items():
+        try:
+            ver = version.parse(metadata.version(lib))
+        except ImportError:
+            return f"snapshot tests require {lib} but it is not installed"
+
+        if ver < version.parse(min_ver):
+            return f"snapshot tests require {lib}>={min_ver} but {ver} is installed"
+
+    return None
+
+
+def pytest_configure(config):
+    if config.option.update_snapshots:
+        from importlib import metadata  # Added in 3.8
+
+        for lib, min_ver in SNAPSHOT_MINIMUM_VERSIONS.items():
+            ver = version.parse(metadata.version(lib))
+            if ver != version.parse(min_ver):
+                pytest.exit(
+                    f"snapshots must be generated with {lib}=={min_ver}"
+                    f" or SNAPSHOT_MINIMUM_VERSIONS must be updated to {ver}"
+                    f" in {__file__}"
+                )
+        return
+
+    reason = _snapshot_skip_reason()
+    if reason:
+        config.issue_config_time_warning(UserWarning(reason), stacklevel=2)
         config.option.warn_unused_snapshots = True
+
+
+def pytest_collection_modifyitems(config, items):
+    reason = _snapshot_skip_reason()
+    if reason:
+        for item in items:
+            if "snap_compare" in item.fixturenames:
+                item.add_marker(pytest.mark.skip(reason=reason))

tests/unit/test_tree_reporter.py

@@ -1,4 +1,3 @@
-import sys
 from dataclasses import dataclass
 from textwrap import dedent
 from typing import Any
@@ -1534,15 +1533,6 @@ def test_render_runs_the_app(self):
 
 @pytest.fixture
 def compare(monkeypatch, tmp_path, snap_compare):
-    # The snapshots we've generated using current versions of Textual aren't
-    # expected to match anymore on Python 3.7, as Textual dropped support for
-    # Python 3.7 in the 0.44 release. However, we'd still like to run our
-    # snapshot tests on Python 3.7, to confirm that no unexpected exceptions
-    # occur and that the app doesn't crash. So, allow `snap_compare()` to drive
-    # the application, but always return `True` on Python 3.7 as long as no
-    # exception was raised.
-    succeed_even_if_mismatched = sys.version_info < (3, 8)
-
     def compare_impl(
         allocations: Iterator[AllocationRecord],
         press: Iterable[str] = (),
@@ -1561,14 +1551,11 @@ def compare_impl(
         with monkeypatch.context() as app_patch:
             app_patch.setitem(globals(), app_global, app)
             tmp_main.write_text(f"from {__name__} import {app_global} as app")
-            return (
-                snap_compare(
-                    str(tmp_main),
-                    press=press,
-                    terminal_size=terminal_size,
-                    run_before=run_before,
-                )
-                or succeed_even_if_mismatched
+            return snap_compare(
+                str(tmp_main),
+                press=press,
+                terminal_size=terminal_size,
+                run_before=run_before,
             )
 
     yield compare_impl

tests/unit/test_tui_reporter.py

@@ -1,6 +1,5 @@
 import asyncio
 import datetime
-import sys
 from io import StringIO
 from typing import Awaitable
 from typing import Callable
@@ -103,15 +102,6 @@ def get_current_snapshot(
 def compare(monkeypatch, tmp_path, snap_compare):
     monkeypatch.setattr(memray.reporters.tui, "datetime", FakeDatetime)
 
-    # The snapshots we've generated using current versions of Textual aren't
-    # expected to match anymore on Python 3.7, as Textual dropped support for
-    # Python 3.7 in the 0.44 release. However, we'd still like to run our
-    # snapshot tests on Python 3.7, to confirm that no unexpected exceptions
-    # occur and that the app doesn't crash. So, allow `snap_compare()` to drive
-    # the application, but always return `True` on Python 3.7 as long as no
-    # exception was raised.
-    succeed_even_if_mismatched = sys.version_info < (3, 8)
-
     def compare_impl(
         cmdline_override: Optional[str] = None,
         press: Iterable[str] = (),
@@ -138,14 +128,11 @@ async def run_before_wrapper(pilot) -> None:
         with monkeypatch.context() as app_patch:
             app_patch.setitem(globals(), app_global, app)
             tmp_main.write_text(f"from {__name__} import {app_global} as app")
-            return (
-                snap_compare(
-                    str(tmp_main),
-                    press=press,
-                    terminal_size=terminal_size,
-                    run_before=run_before_wrapper,
-                )
-                or succeed_even_if_mismatched
+            return snap_compare(
+                str(tmp_main),
+                press=press,
+                terminal_size=terminal_size,
+                run_before=run_before_wrapper,
             )
 
     yield compare_impl