Add support for Python 3.12, add a pinned test env, bump tool versions

.flake8

@@ -1,15 +1,10 @@
 [flake8]
 ignore =
-    # Refers to "whitespace before ':'", which black produces.
-    # https://black.readthedocs.io/en/stable/the_black_code_style/current_style.html#slices
+    # https://black.readthedocs.io/en/latest/guides/using_black_with_other_tools.html#pycodestyle
     E203,
-
-    # Refers to the max-line length. Let's suppress the error and simply
-    # let black take care on how it wants to format the lines.
     E501,
-
-    # Refers to "line break before binary operator".
-    # Similar to above, let black take care of the formatting.
+    E701,
+    E704,
     W503,
 
     # Refers to "necessary dict call - rewrite as a literal".

.github/workflows/publish.yml

@@ -13,9 +13,9 @@ jobs:
     runs-on: ubuntu-latest
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies

.github/workflows/tests-ubuntu.yml

@@ -15,18 +15,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         env:
           - TOXENV: py
         include:
-          - python-version: '3.11'
+          - python-version: '3.8'
             env:
-              TOXENV: old
+              TOXENV: pinned
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -38,7 +38,7 @@ jobs:
       run: |
         tox
     - name: coverage
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       if: ${{ success() }}
       with:
         fail_ci_if_error: false
@@ -48,13 +48,13 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.11']
+        python-version: ['3.12']
         tox-job: ["mypy", "docs", "linters", "types", "twinecheck"]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies

.github/workflows/tests-windows.yml

@@ -15,18 +15,18 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11']
+        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
         env:
           - TOXENV: py
         include:
-          - python-version: '3.11'
+          - python-version: '3.8'
             env:
-              TOXENV: old
+              TOXENV: pinned
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -38,7 +38,7 @@ jobs:
       run: |
         tox
     - name: coverage
-      uses: codecov/codecov-action@v3
+      uses: codecov/codecov-action@v4
       if: ${{ success() }}
       with:
         fail_ci_if_error: false

.pre-commit-config.yaml

@@ -4,13 +4,13 @@ repos:
         language_version: python3
         exclude: ^docs/tutorial-project/
     repo: https://github.com/ambv/black
-    rev: 23.7.0
+    rev: 24.2.0
   - hooks:
       - id: isort
         language_version: python3
         exclude: ^docs/tutorial-project/
     repo: https://github.com/PyCQA/isort
-    rev: 5.12.0
+    rev: 5.13.2
   - hooks:
       - id: flake8
         language_version: python3
@@ -22,4 +22,4 @@ repos:
           - flake8-docstrings
           - flake8-string-format
     repo: https://github.com/pycqa/flake8
-    rev: 6.0.0
+    rev: 7.0.0

setup.py

@@ -20,17 +20,17 @@
     entry_points={"pytest11": ["web-poet = web_poet.testing.pytest"]},
     install_requires=[
         "attrs >= 21.3.0",
-        "parsel",
-        "url-matcher",
-        "multidict",
+        "parsel >= 1.5.0",
+        "url-matcher >= 0.2.0",
+        "multidict >= 0.5.0",
         "w3lib >= 1.22.0",
         "async-lru >= 1.0.3",
         "itemadapter >= 0.8.0",
-        "andi",
-        "python-dateutil",
-        "time-machine",
+        "andi >= 0.5.0",
+        "python-dateutil >= 2.7.0",
+        "time-machine >= 2.7.1",
         "packaging >= 20.0",
-        "backports.zoneinfo; python_version < '3.9' and platform_system != 'Windows'",
+        "backports.zoneinfo >= 0.2.1; python_version < '3.9' and platform_system != 'Windows'",
     ],
     classifiers=[
         "Development Status :: 3 - Alpha",
@@ -43,5 +43,6 @@
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
+        "Programming Language :: Python :: 3.12",
     ],
 )

tests/po_lib/__init__.py

@@ -1,6 +1,7 @@
 """
 This package is just for overrides testing purposes.
 """
+
 from typing import Any, Dict, List, Type, Union
 
 from url_matcher import Patterns
@@ -18,12 +19,10 @@ class POBase(ItemPage):
     expected_meta: Dict[str, Any]
 
 
-class POTopLevelOverriden1(ItemPage):
-    ...
+class POTopLevelOverriden1(ItemPage): ...
 
 
-class POTopLevelOverriden2(ItemPage):
-    ...
+class POTopLevelOverriden2(ItemPage): ...
 
 
 @handle_urls("example.com", instead_of=POTopLevelOverriden1)

tests/po_lib/a_module.py

@@ -4,8 +4,7 @@
 from web_poet import ItemPage, handle_urls
 
 
-class POModuleOverriden(ItemPage):
-    ...
+class POModuleOverriden(ItemPage): ...
 
 
 @handle_urls("example.com", instead_of=POModuleOverriden, extra_arg="foo")

tests/po_lib/nested_package/__init__.py

@@ -4,8 +4,7 @@
 from web_poet import ItemPage, handle_urls
 
 
-class PONestedPkgOverriden(ItemPage):
-    ...
+class PONestedPkgOverriden(ItemPage): ...
 
 
 @handle_urls(

tests/po_lib/nested_package/a_nested_module.py

@@ -4,8 +4,7 @@
 from web_poet import ItemPage, handle_urls
 
 
-class PONestedModuleOverriden(ItemPage):
-    ...
+class PONestedModuleOverriden(ItemPage): ...
 
 
 @handle_urls(

tests/po_lib_sub/__init__.py

@@ -1,6 +1,7 @@
 """This package is being used by tests/po_lib to validate some behaviors on
 external depedencies.
 """
+
 from typing import Any, Dict, Type
 
 from url_matcher import Patterns
@@ -14,8 +15,7 @@ class POBase(ItemPage):
     expected_meta: Dict[str, Any]
 
 
-class POLibSubOverriden(ItemPage):
-    ...
+class POLibSubOverriden(ItemPage): ...
 
 
 @handle_urls("sub.example", instead_of=POLibSubOverriden)

tests/test_input_validation.py

@@ -1,4 +1,5 @@
 """Test page object input validation scenarios."""
+
 from typing import Optional
 
 import attrs

tests/test_rules.py

@@ -465,8 +465,7 @@ def test_handle_urls_deprecation() -> None:
     with pytest.warns(DeprecationWarning, match=msg):
 
         @handle_urls("example.com", overrides=CustomProductPage)
-        class PageWithDeprecatedOverrides:
-            ...
+        class PageWithDeprecatedOverrides: ...
 
     # Despite the deprecation, it should still properly add the rule in the
     # registry.

tests/test_serialization.py

@@ -27,7 +27,7 @@
 
 
 def _assert_pages_equal(p1, p2) -> None:
-    assert type(p1) == type(p2)
+    assert type(p1) == type(p2)  # noqa: E721
     assert type(p1.response) == type(p2.response)  # noqa: E721
     assert type(p1.response.body) == type(p2.response.body)  # noqa: E721
     assert type(p1.response.headers) == type(p2.response.headers)  # noqa: E721
@@ -39,7 +39,7 @@ def _assert_pages_equal(p1, p2) -> None:
 
 
 def _assert_urls_equal(u1: _Url, u2: _Url) -> None:
-    assert type(u1) == type(u2)
+    assert type(u1) == type(u2)  # noqa: E721
     assert str(u1) == str(u2)
 
 
@@ -188,7 +188,7 @@ class MyWebPage(ItemPage):
     read_serialized_deps = storage.read()
     po = MyWebPage(book_list_html_response, url)
     deserialized_po = deserialize(MyWebPage, read_serialized_deps)
-    assert type(deserialized_po) == MyWebPage
+    assert type(deserialized_po) == MyWebPage  # noqa: E721
     _assert_pages_equal(po, deserialized_po)
 
 

tests/test_testing.py

@@ -248,7 +248,7 @@ async def to_item(self) -> Item:
 
 def _assert_frozen_item(
     frozen_time: datetime.datetime,
-    pytester: pytest.Pytester,
+    pytester: "pytest.Pytester",
     response: HttpResponse,
     *,
     outcomes: dict = None,

tests_extra/po_lib_sub_not_imported/__init__.py

@@ -4,6 +4,7 @@
 What we're ultimately trying to test here is to see if the `default_registry`
 captures the rules annotated in this module if it was not imported.
 """
+
 from typing import Any, Dict, Type
 
 from url_matcher import Patterns
@@ -17,8 +18,7 @@ class POBase:
     expected_meta: Dict[str, Any]
 
 
-class POLibSubOverridenNotImported:
-    ...
+class POLibSubOverridenNotImported: ...
 
 
 @handle_urls("sub_not_imported.example", instead_of=POLibSubOverridenNotImported)

tox.ini

@@ -1,16 +1,16 @@
 [tox]
-envlist = py38,py39,py310,py311,mypy,docs,types
+envlist = py38,py39,py310,py311,py312,mypy,docs,types
 
 [pytest]
 asyncio_mode = strict
 
 [testenv]
 deps =
-    pytest
-    pytest-asyncio
+    pytest >= 6.2.0
+    pytest-asyncio >= 0.17.2
     coverage
-    requests
-    aiohttp
+    requests >= 2.18.0
+    aiohttp >= 3.7.0
     zyte-common-items
 
 commands =
@@ -59,11 +59,29 @@ commands =
 deps = -rrequirements-dev.txt
 commands = pre-commit run --all-files --show-diff-on-failure
 
-[testenv:old]
+[testenv:pinned]
+basepython = python3.8
 deps =
     {[testenv]deps}
-    pytest < 7.0
-    parsel < 1.8
+    pytest==6.2.0
+    pytest-asyncio==0.17.2
+    requests==2.18.0
+    zyte-common-items==0.1.0
+    aiohttp==3.7.0
+    attrs==21.3.0
+    parsel==1.5.0
+    url-matcher==0.2.0
+    tldextract==3.0.0
+    multidict==5.0.0
+    w3lib==1.22.0
+    async-lru==1.0.3
+    itemadapter==0.8.0
+    andi==0.5.0
+    python-dateutil==2.7.0
+    time-machine==2.7.1
+    packaging==20.0
+    backports.zoneinfo==0.2.1 ; platform_system != 'Windows'
+
 
 [testenv:twinecheck]
 basepython = python3

web_poet/fields.py

@@ -2,6 +2,7 @@
 ``web_poet.fields`` is a module with helpers for putting extraction logic
 into separate Page Object methods / properties.
 """
+
 import inspect
 from contextlib import suppress
 from functools import update_wrapper, wraps

web_poet/page_inputs/client.py

@@ -211,9 +211,9 @@ async def execute(
             response = await self._request_downloader(request)
         except HttpError as ex:
             if self.save_responses:
-                self._saved_responses[
-                    request_fingerprint(request)
-                ] = _SavedResponseData(request, None, ex)
+                self._saved_responses[request_fingerprint(request)] = (
+                    _SavedResponseData(request, None, ex)
+                )
             raise
 
         if self.save_responses: