Merge pull request #208 from scrapinghub/modernize

Add Python 3.13, drop Python 3.8, update tool versions

Author: kmike

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
     - name: Set up Python
       uses: actions/setup-python@v5
       with:
-        python-version: '3.x'
+        python-version: '3.13'
     - name: Install dependencies
       run: |
         python -m pip install --upgrade pip

.github/workflows/tests-ubuntu.yml

@@ -15,11 +15,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         env:
           - TOXENV: py
         include:
-          - python-version: '3.8'
+          - python-version: '3.9'
             env:
               TOXENV: pinned
 
@@ -48,7 +48,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.12']
+        python-version: ['3.13']
         tox-job: ["mypy", "docs", "linters", "types", "twinecheck"]
 
     steps:

.github/workflows/tests-windows.yml

@@ -15,11 +15,11 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
+        python-version: ['3.9', '3.10', '3.11', '3.12', '3.13']
         env:
           - TOXENV: py
         include:
-          - python-version: '3.8'
+          - python-version: '3.9'
             env:
               TOXENV: pinned
 

.pre-commit-config.yaml

@@ -4,7 +4,7 @@ repos:
         language_version: python3
         exclude: ^docs/tutorial-project/
     repo: https://github.com/ambv/black
-    rev: 24.2.0
+    rev: 24.10.0
   - hooks:
       - id: isort
         language_version: python3
@@ -22,4 +22,4 @@ repos:
           - flake8-docstrings
           - flake8-string-format
     repo: https://github.com/pycqa/flake8
-    rev: 7.0.0
+    rev: 7.1.1

.readthedocs.yml

@@ -6,7 +6,7 @@ sphinx:
 build:
   os: ubuntu-22.04
   tools:
-    python: "3.11"  # Keep in sync with .github/workflows/tests-ubuntu.yml
+    python: "3.13"  # Keep in sync with .github/workflows/tests-ubuntu.yml
 
 python:
   install:

README.rst

@@ -28,7 +28,7 @@ web-poet
 
 .. intro starts
 
-``web-poet`` is a Python 3.8+ implementation of the `page object pattern`_ for
+``web-poet`` is a Python 3.9+ implementation of the `page object pattern`_ for
 web scraping. It enables writing portable, reusable web parsing code.
 
 .. _page object pattern: https://martinfowler.com/bliki/PageObject.html

docs/conf.py

@@ -7,8 +7,6 @@
 import os
 import sys
 
-import sphinx_rtd_theme
-
 # -- Path setup --------------------------------------------------------------
 
 # If extensions (or modules to document with autodoc) are in another directory,
@@ -54,7 +52,7 @@
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = ".rst"
+source_suffix = {".rst": "restructuredtext"}
 
 # The master toctree document.
 master_doc = "index"
@@ -82,12 +80,6 @@
 #
 html_theme = "sphinx_rtd_theme"
 
-# Add any paths that contain custom themes here, relative to this directory.
-# Add path to the RTD explicitly to robustify builds (otherwise might
-# fail in a clean Debian build env)
-
-html_theme_path = [sphinx_rtd_theme.get_html_theme_path()]
-
 # Theme options are theme-specific and customize the look and feel of a theme
 # further.  For a list of options available for each theme, see the
 # documentation.

docs/requirements.txt

@@ -1,2 +1,2 @@
-Sphinx==5.3.0
-sphinx-rtd-theme==1.0.0
+Sphinx==8.1.3
+sphinx-rtd-theme==3.0.1

setup.py

@@ -30,7 +30,6 @@
         "python-dateutil >= 2.7.0",
         "time-machine >= 2.7.1",
         "packaging >= 20.0",
-        "backports.zoneinfo >= 0.2.1; python_version < '3.9' and platform_system != 'Windows'",
     ],
     classifiers=[
         "Development Status :: 3 - Alpha",
@@ -39,10 +38,10 @@
         "Natural Language :: English",
         "Operating System :: OS Independent",
         "Programming Language :: Python :: 3",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
+        "Programming Language :: Python :: 3.13",
     ],
 )

tests/test_serialization.py

@@ -1,5 +1,4 @@
-import sys
-from typing import Type
+from typing import Annotated, Type
 
 import attrs
 import pytest
@@ -218,12 +217,7 @@ def test_httpclient_empty(tmp_path) -> None:
     assert "HttpClient" in read_serialized_deps
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 def test_annotated(book_list_html_response) -> None:
-    from typing import Annotated
-
     @attrs.define
     class MyWebPage(ItemPage):
         response: Annotated[HttpResponse, "foo", 42]
@@ -243,9 +237,6 @@ class MyWebPage(ItemPage):
     _assert_pages_equal(po, deserialized_po)
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 def test_annotated_duplicate(book_list_html_response) -> None:
     url_str = "http://books.toscrape.com/index.html"
     url = ResponseUrl(url_str)

tests/test_testing.py

@@ -1,9 +1,9 @@
 import datetime
 import json
-import sys
 from collections import deque
 from pathlib import Path
-from typing import Any, Dict, Optional
+from typing import Annotated, Any, Dict, Optional
+from zoneinfo import ZoneInfo
 
 import attrs
 import dateutil.tz
@@ -223,7 +223,8 @@ class MetadataLocalTime(Metadata):
 
 @attrs.define(kw_only=True)
 class ProductLocalTime(Product):
-    metadata: Optional[MetadataLocalTime]
+    # in newer zyte-common-items this should inherit from ProductMetadata
+    metadata: Optional[MetadataLocalTime]  # type: ignore[assignment]
 
 
 def _get_product_item(date: datetime.datetime) -> ProductLocalTime:
@@ -282,11 +283,6 @@ def test_pytest_frozen_time_naive(pytester, book_list_html_response) -> None:
 @pytest.mark.skipif(not time_machine.HAVE_TZSET, reason="Not supported on Windows")
 @pytest.mark.parametrize("offset", [-5, 0, 8])
 def test_pytest_frozen_time_tz(pytester, book_list_html_response, offset) -> None:
-    if sys.version_info >= (3, 9):
-        from zoneinfo import ZoneInfo
-    else:
-        from backports.zoneinfo import ZoneInfo
-
     tzinfo = ZoneInfo(f"Etc/GMT{-offset:+d}")
     frozen_time = datetime.datetime(2022, 3, 4, 20, 21, 22, tzinfo=tzinfo)
     _assert_frozen_item(frozen_time, pytester, book_list_html_response)
@@ -540,20 +536,14 @@ def test_page_object_exception_none(pytester, book_list_html_response) -> None:
     result.assert_outcomes(failed=1)
 
 
-if sys.version_info >= (3, 9):
-    from typing import Annotated
-
-    @attrs.define(kw_only=True)
-    class MyAnnotatedItemPage(MyItemPage):
-        response: Annotated[HttpResponse, "foo", 42]
+@attrs.define(kw_only=True)
+class MyAnnotatedItemPage(MyItemPage):
+    response: Annotated[HttpResponse, "foo", 42]
 
-        async def to_item(self) -> dict:
-            return {"foo": "bar"}
+    async def to_item(self) -> dict:
+        return {"foo": "bar"}
 
 
-@pytest.mark.skipif(
-    sys.version_info < (3, 9), reason="No Annotated support in Python < 3.9"
-)
 def test_annotated(pytester, book_list_html_response) -> None:
     _save_fixture(
         pytester,

tox.ini

@@ -1,5 +1,5 @@
 [tox]
-envlist = py38,py39,py310,py311,py312,mypy,docs,types
+envlist = py39,py310,py311,py312,py313,mypy,docs,types
 
 [pytest]
 asyncio_mode = strict
@@ -31,9 +31,13 @@ commands =
 
 [testenv:mypy]
 deps =
-    mypy==1.8.0
+    mypy==1.11.2
+    aiohttp
+    py
+    pytest
     types-requests
     types-python-dateutil
+    zyte-common-items
 
 commands = mypy web_poet tests
 
@@ -58,7 +62,7 @@ deps = -rrequirements-dev.txt
 commands = pre-commit run --all-files --show-diff-on-failure
 
 [testenv:pinned]
-basepython = python3.8
+basepython = python3.9
 deps =
     {[testenv]deps}
     pytest==6.2.0
@@ -78,14 +82,13 @@ deps =
     python-dateutil==2.7.0
     time-machine==2.7.1
     packaging==20.0
-    backports.zoneinfo==0.2.1 ; platform_system != 'Windows'
 
 
 [testenv:twinecheck]
 basepython = python3
 deps =
     twine==5.1.1
-    build==0.10.0
+    build==1.2.2
 commands =
     python -m build --sdist
     twine check dist/*

web_poet/fields.py

@@ -6,7 +6,7 @@
 import inspect
 from contextlib import suppress
 from functools import update_wrapper, wraps
-from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar
+from typing import Callable, Dict, List, Optional, Tuple, Type, TypeVar, cast
 
 import attrs
 from itemadapter import ItemAdapter
@@ -163,7 +163,7 @@ def processed(page):
     if method is not None:
         # @field syntax
         res = _field(method)
-        update_wrapper(res, method)
+        update_wrapper(cast(Callable, res), method)
         return res
     else:
         # @field(...) syntax

web_poet/testing/fixture.py

@@ -3,9 +3,9 @@
 import json
 import logging
 import os
-import sys
 from pathlib import Path
 from typing import Any, Iterable, Optional, Type, TypeVar, Union, cast
+from zoneinfo import ZoneInfo
 
 import dateutil.parser
 import dateutil.tz
@@ -181,11 +181,6 @@ def _parse_frozen_time(meta_value: str) -> datetime.datetime:
             )
             return parsed_value.astimezone()
 
-        if sys.version_info >= (3, 9):
-            from zoneinfo import ZoneInfo
-        else:
-            from backports.zoneinfo import ZoneInfo
-
         if parsed_value.tzinfo == dateutil.tz.UTC:
             return parsed_value.replace(tzinfo=ZoneInfo("UTC"))
 

web_poet/testing/pytest.py

@@ -71,15 +71,15 @@ def collect(self) -> Iterable[Union[pytest.Item, pytest.Collector]]:
                 WebPoetItem.from_parent(parent=self, name="item", fixture=self.fixture)
             ]
         else:
-            overall_tests = [
+            overall_tests: List[pytest.Item] = [
                 WebPoetNoToItemException.from_parent(
                     parent=self, name="TO_ITEM_DOESNT_RAISE", fixture=self.fixture
                 ),
                 WebPoetNoExtraFieldsItem.from_parent(
                     parent=self, name="NO_EXTRA_FIELDS", fixture=self.fixture
                 ),
             ]
-            field_tests = [
+            field_tests: List[pytest.Item] = [
                 WebPoetFieldItem.from_parent(
                     parent=self, name=field, fixture=self.fixture, field_name=field
                 )