From 34c277b32d50f83d7e0fe74f7d4995358c09a4f1 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 9 Aug 2024 11:58:39 +0100 Subject: [PATCH 1/4] Use uv for managing pinned dev dependencies Prior to this change, development virtual environments were provisioned with `pip` (a relatively slow tool) with dependency versions solved dynamically using spec in 'pyproject.toml' (making each build something of a lottery). This change replaces `pip` with the far faster `uv` [^1], and uses pinned versions of dependencies in 'requirements.txt' (making environments reproducible). [^1]: https://github.com/astral-sh/uv --- docs/xocto/development.md | 9 ++ makefile | 6 +- requirements.txt | 300 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 313 insertions(+), 2 deletions(-) create mode 100644 requirements.txt diff --git a/docs/xocto/development.md b/docs/xocto/development.md index 156d819..60499bd 100644 --- a/docs/xocto/development.md +++ b/docs/xocto/development.md @@ -46,6 +46,15 @@ of this workflow: This is why typically in shared projects version releases are separated into their own pull requests. +## Altering dependencies + +* Edit `pyproject.toml`. +* `make sync_requirements` to refresh `requirements.txt`. +* `make install test format_check lint_check mypy`. + +When you are ready, commit both `pyproject.toml` and +`requirements.txt`. + ## Publishing ### Version number diff --git a/makefile b/makefile index a2bc22f..94279d6 100644 --- a/makefile +++ b/makefile @@ -1,7 +1,9 @@ install: - pip install pip==23.3.1 - pip install -e '.[dev,docs]' + pip install uv==0.2.34 + uv pip install -r requirements.txt --editable . +sync_requirements: + uv pip compile --output-file=requirements.txt --extra=dev pyproject.toml # CI step wrappers diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..70997ff --- /dev/null +++ b/requirements.txt @@ -0,0 +1,300 @@ +# This file was autogenerated by uv via the following command: +# uv pip compile --output-file=requirements.txt --extra=dev pyproject.toml +annotated-types==0.7.0 + # via pydantic +anyio==4.4.0 + # via starlette +asgiref==3.8.1 + # via django +attrs==24.2.0 + # via + # cattrs + # ddtrace + # hypothesis +backports-tarfile==1.2.0 + # via jaraco-context +boto3==1.26.53 + # via + # xocto (pyproject.toml) + # moto +botocore==1.29.53 + # via + # xocto (pyproject.toml) + # boto3 + # moto + # s3transfer +bytecode==0.15.1 + # via ddtrace +cattrs==23.2.3 + # via ddtrace +certifi==2024.7.4 + # via requests +cffi==1.17.0 + # via + # cryptography + # pact-python +cfgv==3.4.0 + # via pre-commit +charset-normalizer==3.3.2 + # via requests +click==8.1.7 + # via + # pact-python + # uvicorn +cryptography==43.0.0 + # via moto +ddsketch==3.0.1 + # via ddtrace +ddtrace==2.10.4 + # via xocto (pyproject.toml) +deprecated==1.2.14 + # via opentelemetry-api +distlib==0.3.8 + # via virtualenv +django==4.2.15 + # via xocto (pyproject.toml) +docutils==0.21.2 + # via readme-renderer +duckdb==1.0.0 + # via xocto (pyproject.toml) +envier==0.5.2 + # via ddtrace +et-xmlfile==1.1.0 + # via openpyxl +exceptiongroup==1.2.2 + # via + # anyio + # cattrs + # hypothesis + # pytest +fastapi==0.112.0 + # via pact-python +filelock==3.15.4 + # via virtualenv +h11==0.14.0 + # via uvicorn +hypothesis==6.62.1 + # via xocto (pyproject.toml) +identify==2.6.0 + # via pre-commit +idna==3.7 + # via + # anyio + # requests + # yarl +importlib-metadata==8.2.0 + # via + # keyring + # twine +iniconfig==2.0.0 + # via pytest +jaraco-classes==3.4.0 + # via keyring +jaraco-context==5.3.0 + # via keyring +jaraco-functools==4.0.2 + # via keyring +jinja2==3.1.4 + # via moto +jmespath==1.0.1 + # via + # boto3 + # botocore +keyring==25.3.0 + # via twine +markdown-it-py==3.0.0 + # via rich +markupsafe==2.1.5 + # via + # jinja2 + # werkzeug +mdurl==0.1.2 + # via markdown-it-py +more-itertools==10.4.0 + # via + # jaraco-classes + # jaraco-functools +moto==4.1.0 + # via xocto (pyproject.toml) +multidict==6.0.5 + # via yarl +mypy==1.10.0 + # via xocto (pyproject.toml) +mypy-boto3-s3==1.34.120 + # via xocto (pyproject.toml) +mypy-extensions==1.0.0 + # via mypy +nh3==0.2.18 + # via readme-renderer +nodeenv==1.9.1 + # via pre-commit +numpy==1.22.2 + # via + # xocto (pyproject.toml) + # pandas + # pyarrow +openpyxl==3.1.5 + # via xocto (pyproject.toml) +opentelemetry-api==1.16.0 + # via ddtrace +packaging==24.1 + # via pytest +pact-python==2.2.1 + # via xocto (pyproject.toml) +pandas==2.0.3 + # via xocto (pyproject.toml) +pkginfo==1.11.1 + # via twine +platformdirs==4.2.2 + # via virtualenv +pluggy==1.5.0 + # via pytest +pre-commit==3.8.0 + # via xocto (pyproject.toml) +protobuf==5.27.3 + # via ddtrace +psutil==6.0.0 + # via pact-python +psycopg2==2.9.9 + # via xocto (pyproject.toml) +pyarrow==17.0.0 + # via xocto (pyproject.toml) +pyarrow-stubs==10.0.1.6 + # via xocto (pyproject.toml) +pycparser==2.22 + # via cffi +pydantic==2.8.2 + # via fastapi +pydantic-core==2.20.1 + # via pydantic +pygments==2.18.0 + # via + # readme-renderer + # rich +pytest==8.0.2 + # via + # xocto (pyproject.toml) + # pytest-django + # pytest-mock +pytest-django==4.8.0 + # via xocto (pyproject.toml) +pytest-mock==3.12.0 + # via xocto (pyproject.toml) +python-dateutil==2.9.0.post0 + # via + # xocto (pyproject.toml) + # botocore + # moto + # pandas + # time-machine +python-magic==0.4.27 + # via xocto (pyproject.toml) +pytz==2024.1 + # via + # xocto (pyproject.toml) + # pandas +pyyaml==6.0.2 + # via + # moto + # pre-commit + # responses +readme-renderer==44.0 + # via twine +requests==2.32.3 + # via + # moto + # pact-python + # requests-toolbelt + # responses + # twine +requests-toolbelt==1.0.0 + # via twine +responses==0.25.3 + # via moto +rfc3986==2.0.0 + # via twine +rich==13.7.1 + # via twine +ruff==0.4.9 + # via xocto (pyproject.toml) +s3transfer==0.6.2 + # via boto3 +setuptools==72.1.0 + # via opentelemetry-api +six==1.16.0 + # via + # ddsketch + # ddtrace + # pact-python + # python-dateutil +sniffio==1.3.1 + # via anyio +sortedcontainers==2.4.0 + # via hypothesis +sqlparse==0.5.1 + # via django +starlette==0.37.2 + # via fastapi +structlog==24.4.0 + # via xocto (pyproject.toml) +time-machine==2.14.1 + # via xocto (pyproject.toml) +tomli==2.0.1 + # via + # mypy + # pytest +twine==4.0.2 + # via xocto (pyproject.toml) +types-openpyxl==3.0.4.5 + # via xocto (pyproject.toml) +types-python-dateutil==2.8.19.20240106 + # via xocto (pyproject.toml) +types-pytz==2024.1.0.20240203 + # via xocto (pyproject.toml) +types-requests==2.28.11.8 + # via xocto (pyproject.toml) +types-urllib3==1.26.25.14 + # via types-requests +typing-extensions==4.12.2 + # via + # anyio + # asgiref + # bytecode + # cattrs + # ddtrace + # fastapi + # mypy + # mypy-boto3-s3 + # pact-python + # pydantic + # pydantic-core + # starlette + # uvicorn +tzdata==2024.1 + # via pandas +urllib3==1.26.19 + # via + # botocore + # requests + # responses + # twine +uvicorn==0.30.5 + # via pact-python +virtualenv==20.26.3 + # via pre-commit +werkzeug==3.0.3 + # via moto +wheel==0.38.4 + # via xocto (pyproject.toml) +wrapt==1.16.0 + # via deprecated +xlrd==2.0.1 + # via xocto (pyproject.toml) +xmltodict==0.13.0 + # via + # ddtrace + # moto +yarl==1.9.4 + # via pact-python +zipp==3.19.2 + # via importlib-metadata From b383b02561bb5cc5fac8b42cac1a55e629fa07e9 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 9 Aug 2024 12:08:33 +0100 Subject: [PATCH 2/4] Don't pin versions of 'dev' dependencies in 'pyproject.toml' Prior to this change, `pip install xocto[dev]` would use pinned versions of 'dev' dependencies This change removes pinned versions from 'dev' dependencies in 'pyproject.toml'. Package maintainers should provision virtual environments using versions in 'requirements.txt' with `uv pip sync requirements.txt`. --- pyproject.toml | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 4a77554..bd9dae9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -41,27 +41,27 @@ dependencies = [ [project.optional-dependencies] dev = [ - "boto3==1.26.53", - "botocore==1.29.53", - "hypothesis==6.62.1", - "moto[s3,sqs]==4.1", - "mypy-boto3-s3==1.34.120", - "mypy==1.10.0", - "numpy==1.22.2", - "pre-commit>=3.7.1", - "psycopg2>=2.8.4", - "pyarrow-stubs==10.0.1.6", - "pytest-django==4.8.0", - "pytest-mock==3.12.0", - "pytest==8.0.2", - "ruff==0.4.9", - "time-machine==2.14.1", - "twine==4.0.2", - "types-openpyxl==3.0.4.5", - "types-python-dateutil==2.8.19.20240106", - "types-pytz==2024.1.0.20240203", - "types-requests==2.28.11.8", - "wheel==0.38.4", + "boto3", + "botocore", + "hypothesis", + "moto[s3,sqs]", + "mypy-boto3-s3", + "mypy", + "numpy", + "pre-commit", + "psycopg2", + "pyarrow-stubs", + "pytest-django", + "pytest-mock", + "pytest", + "ruff", + "time-machine", + "twine", + "types-openpyxl", + "types-python-dateutil", + "types-pytz", + "types-requests", + "wheel", ] docs = [ "Sphinx==7.2.6", From 7fb9540e9f768d576a6a6771834cd9d62f16fbef Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 9 Aug 2024 13:45:54 +0100 Subject: [PATCH 3/4] Add support for 'tox' Adding 'tox' [^1] makes it possible to test xocto against multiple versions of Python. [^1]: https://tox.wiki/ --- docs/xocto/development.md | 11 +++++++++++ pyproject.toml | 2 ++ requirements.txt | 41 ++++++++++++++++++++++++++++++++++----- tox.ini | 15 ++++++++++++++ 4 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 tox.ini diff --git a/docs/xocto/development.md b/docs/xocto/development.md index 60499bd..a64d0bf 100644 --- a/docs/xocto/development.md +++ b/docs/xocto/development.md @@ -55,6 +55,17 @@ their own pull requests. When you are ready, commit both `pyproject.toml` and `requirements.txt`. +## tox + +Run [tox](https://tox.wiki/) to test the package on supported versions +of Python. + +Each versions of Python listed in `tox.ini` must be present on your +system. On Linux it is probably easiest to build each version from +source, using `make altinstall`. +If you use Mac you can install each version using homebrew e.g., +`brew install python@3.9`. + ## Publishing ### Version number diff --git a/pyproject.toml b/pyproject.toml index bd9dae9..fe2b0a7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,8 @@ dev = [ "pytest", "ruff", "time-machine", + "tox", + "tox-uv", "twine", "types-openpyxl", "types-python-dateutil", diff --git a/requirements.txt b/requirements.txt index 70997ff..50ad086 100644 --- a/requirements.txt +++ b/requirements.txt @@ -25,6 +25,8 @@ botocore==1.29.53 # s3transfer bytecode==0.15.1 # via ddtrace +cachetools==5.4.0 + # via tox cattrs==23.2.3 # via ddtrace certifi==2024.7.4 @@ -35,12 +37,16 @@ cffi==1.17.0 # pact-python cfgv==3.4.0 # via pre-commit +chardet==5.2.0 + # via tox charset-normalizer==3.3.2 # via requests click==8.1.7 # via # pact-python # uvicorn +colorama==0.4.6 + # via tox cryptography==43.0.0 # via moto ddsketch==3.0.1 @@ -70,7 +76,9 @@ exceptiongroup==1.2.2 fastapi==0.112.0 # via pact-python filelock==3.15.4 - # via virtualenv + # via + # tox + # virtualenv h11==0.14.0 # via uvicorn hypothesis==6.62.1 @@ -138,7 +146,11 @@ openpyxl==3.1.5 opentelemetry-api==1.16.0 # via ddtrace packaging==24.1 - # via pytest + # via + # pyproject-api + # pytest + # tox + # tox-uv pact-python==2.2.1 # via xocto (pyproject.toml) pandas==2.0.3 @@ -146,9 +158,13 @@ pandas==2.0.3 pkginfo==1.11.1 # via twine platformdirs==4.2.2 - # via virtualenv + # via + # tox + # virtualenv pluggy==1.5.0 - # via pytest + # via + # pytest + # tox pre-commit==3.8.0 # via xocto (pyproject.toml) protobuf==5.27.3 @@ -171,6 +187,8 @@ pygments==2.18.0 # via # readme-renderer # rich +pyproject-api==1.7.1 + # via tox pytest==8.0.2 # via # xocto (pyproject.toml) @@ -242,7 +260,15 @@ time-machine==2.14.1 tomli==2.0.1 # via # mypy + # pyproject-api # pytest + # tox +tox==4.17.1 + # via + # xocto (pyproject.toml) + # tox-uv +tox-uv==1.11.1 + # via xocto (pyproject.toml) twine==4.0.2 # via xocto (pyproject.toml) types-openpyxl==3.0.4.5 @@ -269,6 +295,7 @@ typing-extensions==4.12.2 # pydantic # pydantic-core # starlette + # tox-uv # uvicorn tzdata==2024.1 # via pandas @@ -278,10 +305,14 @@ urllib3==1.26.19 # requests # responses # twine +uv==0.2.34 + # via tox-uv uvicorn==0.30.5 # via pact-python virtualenv==20.26.3 - # via pre-commit + # via + # pre-commit + # tox werkzeug==3.0.3 # via moto wheel==0.38.4 diff --git a/tox.ini b/tox.ini new file mode 100644 index 0000000..46a3941 --- /dev/null +++ b/tox.ini @@ -0,0 +1,15 @@ +[tox] +min_version = 4.0 +envlist = py39 + +[testenv] +# Install wheels instead of source distributions for faster execution. +package = wheel +# Share the build environment between tox environments. +wheel_build_env = .pkg + +# Run type checking because 'typing' and related packages vary among versions of Python +allowlist_externals = make +commands = + uv pip sync requirements.txt + make test mypy From 615a08031c9bf5c0c8d0689234d08d549107d2d3 Mon Sep 17 00:00:00 2001 From: Lawrence Hudson Date: Fri, 9 Aug 2024 13:52:57 +0100 Subject: [PATCH 4/4] Make all tests pass on Python 3.9 to 3.12 inclusive This change adds Python 3.10, 3.11, and 3.12 to 'tox.ini', and makes changes to 'dev' dependencies, and to pytest filter warnings, such that tests pass on Python 3.9 to 3.12 inclusive. --- pyproject.toml | 7 +++++-- requirements.txt | 4 ++-- tox.ini | 2 +- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index fe2b0a7..217451e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,6 +22,7 @@ classifiers = [ "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", ] license = {text = "MIT"} dependencies = [ @@ -47,10 +48,10 @@ dev = [ "moto[s3,sqs]", "mypy-boto3-s3", "mypy", - "numpy", + "numpy>=1.26.4", "pre-commit", "psycopg2", - "pyarrow-stubs", + "pyarrow-stubs>=10.0.1.9", "pytest-django", "pytest-mock", "pytest", @@ -209,6 +210,8 @@ section-order = [ filterwarnings = [ "error::RuntimeWarning", "error::DeprecationWarning", + "ignore:datetime.datetime.utcfromtimestamp\\(\\) is deprecated and scheduled for removal in a future version:DeprecationWarning", + "ignore:datetime.datetime.utcnow\\(\\) is deprecated and scheduled for removal in a future version:DeprecationWarning", "ignore:defusedxml.lxml:DeprecationWarning:zeep", "ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated since Python 3.3:DeprecationWarning:(graphene|singledispatch)", # https://github.com/ktosiek/pytest-freezegun/issues/35 diff --git a/requirements.txt b/requirements.txt index 50ad086..03b4634 100644 --- a/requirements.txt +++ b/requirements.txt @@ -136,7 +136,7 @@ nh3==0.2.18 # via readme-renderer nodeenv==1.9.1 # via pre-commit -numpy==1.22.2 +numpy==1.26.4 # via # xocto (pyproject.toml) # pandas @@ -175,7 +175,7 @@ psycopg2==2.9.9 # via xocto (pyproject.toml) pyarrow==17.0.0 # via xocto (pyproject.toml) -pyarrow-stubs==10.0.1.6 +pyarrow-stubs==10.0.1.9 # via xocto (pyproject.toml) pycparser==2.22 # via cffi diff --git a/tox.ini b/tox.ini index 46a3941..4385783 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] min_version = 4.0 -envlist = py39 +envlist = py3{9,10,11,12} [testenv] # Install wheels instead of source distributions for faster execution.