From 3bd0e30ae42f26a96edd790a2d52e699463c185b Mon Sep 17 00:00:00 2001 From: Sage Abdullah Date: Mon, 27 Jan 2025 05:03:09 +0000 Subject: [PATCH 1/4] Bump default Python version to 3.12 --- .env | 2 +- Containerfile | 2 +- README.md | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.env b/.env index e77a955..b48d9a6 100644 --- a/.env +++ b/.env @@ -1,5 +1,5 @@ PYTHON_IMPLEMENTATION=python -PYTHON_VERSION=3.10 +PYTHON_VERSION=3.12 MARIADB_VERSION=10.5 MYSQL_VERSION=8.0 ORACLE_VERSION=23.5.0.0 diff --git a/Containerfile b/Containerfile index c75ffeb..8c5be34 100644 --- a/Containerfile +++ b/Containerfile @@ -1,7 +1,7 @@ # syntax=docker/dockerfile:1.12 ARG PYTHON_IMPLEMENTATION=python -ARG PYTHON_VERSION=3.10 +ARG PYTHON_VERSION=3.12 FROM ${PYTHON_IMPLEMENTATION}:${PYTHON_VERSION}-slim-bookworm LABEL org.opencontainers.image.authors="Django Software Foundation" diff --git a/README.md b/README.md index a64b14e..c073140 100644 --- a/README.md +++ b/README.md @@ -193,7 +193,7 @@ environment variables: | ----------------------- | ------------- | ---------------------------------------------------- | | `DJANGO_PATH` | `../django` | Path to the Django repostory on your local machine | | `PYTHON_IMPLEMENTATION` | `python` | Implementation of Python to use — `python` or `pypy` | -| `PYTHON_VERSION` | `3.10` | Version of Python container image to use | +| `PYTHON_VERSION` | `3.12` | Version of Python container image to use | The versions of various backend services can be switched by setting these environment variables: @@ -212,7 +212,7 @@ The `PYTHON_VERSION` environment variable controls which version of Python you are running the tests against, e.g. ```console -$ PYTHON_VERSION=3.10 docker compose run --rm sqlite +$ PYTHON_VERSION=3.12 docker compose run --rm sqlite ``` In addition, it's possible to select a different implementation of Python, i.e. From d584e914d63d268c66549ac6c4dbdfa445f6751d Mon Sep 17 00:00:00 2001 From: Sage Abdullah Date: Tue, 28 Jan 2025 20:23:12 +0000 Subject: [PATCH 2/4] Added sqlite-lib service to test against specific SQLite versions. --- .env | 4 ++++ README.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++- compose.yml | 39 +++++++++++++++++++++++++++++++++++++-- packages.txt | 1 + 4 files changed, 89 insertions(+), 3 deletions(-) diff --git a/.env b/.env index b48d9a6..e7a0a50 100644 --- a/.env +++ b/.env @@ -5,3 +5,7 @@ MYSQL_VERSION=8.0 ORACLE_VERSION=23.5.0.0 POSTGRESQL_VERSION=14 POSTGIS_VERSION=3.1 +SQLITE_VERSION=3.31.0 +SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE \ + -DSQLITE_ENABLE_JSON1 \ + -DSQLITE_MAX_VARIABLE_NUMBER=32766" diff --git a/README.md b/README.md index c073140..d2036c8 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,7 @@ $ docker compose run --rm mysql $ docker compose run --rm oracle $ docker compose run --rm postgres $ docker compose run --rm sqlite +$ docker compose run --rm sqlite-lib ``` Each of the above commands will run the test suite for a different supported @@ -204,7 +205,14 @@ The versions of various backend services can be switched by setting these enviro | `ORACLE_VERSION` | `23.5.0.0` | Version of Oracle container image to use | | `POSTGRESQL_VERSION` | `14` | Version of PostgreSQL container image to use | | `POSTGIS_VERSION` | `3.1` | Version of PostGIS extension to use | +| `SQLITE_VERSION` | `3.31.0` | Version of SQLite to compile and use | +> [!NOTE] +> +> Using a specific SQLite version requires compiling it from source. To +> customize the `CFLAGS` used for the compilation, you can set the +> `SQLITE_CFLAGS` environment variable. See the [`.env`][10] file for its +> default value. For more details, see [SQLite Versions](#SQLite-Versions). ### Python Versions @@ -229,7 +237,8 @@ restrictions with respect to the range of versions available. ### Database Versions Most database container images are pulled from [Docker Hub][2]. Oracle database -is pulled from the [Oracle Container Registry][3]. +is pulled from the [Oracle Container Registry][3]. Specific versions of SQLite +are compiled directly from the tags in the [official Git mirror][11]. You can switch the version of the database you test against by changing the appropriate environment variable. Available options and their defaults can be @@ -273,6 +282,41 @@ To determine what database versions can be used you can check the release notes for the branch of Django that you have checked out, or alternatively there is the [supported database versions][4] page on Django's Trac Wiki. +#### SQLite Versions + +SQLite is normally bundled in the Python installation using the version +available on the system where Python is compiled. We use the Python Docker image +based on Debian `bookworm`, which has SQLite 3.40.1. + +To use a different version, we compile SQLite from source and load the library +dynamically using `LD_PRELOAD`. There are a few caveats as a result: + +- Some SQLite features are only available if certain flags are set during + compilation. SQLite is known to change these flags in newer releases, such as + to enable features by default that were previously opt-in. When Python is + compiled, it inspects the system's SQLite to determine features that are + included in the `sqlite` module. A mismatch in the module and the dynamically + loaded library may result in Python failing to load, which may happen if we + use an SQLite version that is older than the system version. +- Debian and Ubuntu use a custom `CFLAGS` variable to compile their distributed + SQLite. Historically, Django's CI has only been configured with SQLite + versions that come with the operating system. If SQLite is compiled with + different flags, some tests may fail. + +We currently work around the above caveats by setting the simplest `CFLAGS` +value that allows all the tests to pass. In the future, the Django codebase may +be more robust when tested against the different SQLite configurations and these +workarounds may no longer be necessary. + +Running the tests against a specific SQLite version must be done using the +`sqlite-lib` container instead of `sqlite`. + +```console +$ docker compose run --rm sqlite-lib +``` + +This is done to avoid compiling SQLite when you are not testing against a +specific version. ### Other Versions @@ -309,3 +353,5 @@ with no promises that they'll be delivered: [7]: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/unit-tests/#running-the-unit-tests [8]: https://docs.djangoproject.com/en/stable/internals/contributing/writing-code/unit-tests/#running-the-selenium-tests [9]: https://docs.djangoproject.com/en/stable/ref/contrib/gis/testing/#geodjango-tests +[10]: .env +[11]: https://github.com/sqlite/sqlite diff --git a/compose.yml b/compose.yml index fc65ac7..baa042d 100644 --- a/compose.yml +++ b/compose.yml @@ -8,7 +8,7 @@ x-base: &base args: - PYTHON_IMPLEMENTATION=${PYTHON_IMPLEMENTATION} - PYTHON_VERSION=${PYTHON_VERSION} - additional_contexts: + additional_contexts: &additional-contexts src: ${DJANGO_PATH:-../django} volumes: - ${DJANGO_PATH:-../django}:/django/source:rw @@ -82,6 +82,36 @@ x-postgresql-base: &postgresql-base -c wal_level=minimal # 13+: -c wal_keep_size=0 +x-sqlite-lib-base: &sqlite-lib-base + image: django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION}-sqlite${SQLITE_VERSION} + pull_policy: never + build: + context: . + dockerfile_inline: | + FROM django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION} + SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] + # Only compile SQLite and set LD_PRELOAD if a version is specified. + RUN < Date: Tue, 28 Jan 2025 22:18:00 +0000 Subject: [PATCH 3/4] Consolidated sqlite-lib service into sqlite. --- .env | 2 +- README.md | 37 +++++++++++++++------------- compose.yml | 69 +++++++++++++++++++++++++---------------------------- 3 files changed, 54 insertions(+), 54 deletions(-) diff --git a/.env b/.env index e7a0a50..c1c4223 100644 --- a/.env +++ b/.env @@ -5,7 +5,7 @@ MYSQL_VERSION=8.0 ORACLE_VERSION=23.5.0.0 POSTGRESQL_VERSION=14 POSTGIS_VERSION=3.1 -SQLITE_VERSION=3.31.0 +SQLITE_VERSION= SQLITE_CFLAGS="-DSQLITE_ENABLE_DESERIALIZE \ -DSQLITE_ENABLE_JSON1 \ -DSQLITE_MAX_VARIABLE_NUMBER=32766" diff --git a/README.md b/README.md index d2036c8..dba7dfe 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ Tooling and test execution support for [Django][0] :unicorn: 3. Build the image: ```console - $ docker compose build sqlite + $ docker compose build base ``` 4. Run the tests: @@ -66,7 +66,6 @@ $ docker compose run --rm mysql $ docker compose run --rm oracle $ docker compose run --rm postgres $ docker compose run --rm sqlite -$ docker compose run --rm sqlite-lib ``` Each of the above commands will run the test suite for a different supported @@ -205,14 +204,13 @@ The versions of various backend services can be switched by setting these enviro | `ORACLE_VERSION` | `23.5.0.0` | Version of Oracle container image to use | | `POSTGRESQL_VERSION` | `14` | Version of PostgreSQL container image to use | | `POSTGIS_VERSION` | `3.1` | Version of PostGIS extension to use | -| `SQLITE_VERSION` | `3.31.0` | Version of SQLite to compile and use | +| `SQLITE_VERSION` | | Version of SQLite to compile and use | > [!NOTE] > -> Using a specific SQLite version requires compiling it from source. To -> customize the `CFLAGS` used for the compilation, you can set the -> `SQLITE_CFLAGS` environment variable. See the [`.env`][10] file for its -> default value. For more details, see [SQLite Versions](#SQLite-Versions). +> If left unspecified, the SQLite version provided by Debian will be used. +> Using a specific SQLite version requires compiling it from source. For more +> details, see [SQLite Versions](#SQLite-Versions). ### Python Versions @@ -304,19 +302,24 @@ dynamically using `LD_PRELOAD`. There are a few caveats as a result: different flags, some tests may fail. We currently work around the above caveats by setting the simplest `CFLAGS` -value that allows all the tests to pass. In the future, the Django codebase may -be more robust when tested against the different SQLite configurations and these -workarounds may no longer be necessary. +value that allows all the tests to pass. To customize the `CFLAGS` used for the +compilation, you can set the `SQLITE_CFLAGS` environment variable. See the +[`.env`][10] file for its default value. -Running the tests against a specific SQLite version must be done using the -`sqlite-lib` container instead of `sqlite`. - -```console -$ docker compose run --rm sqlite-lib +``` +SQLITE_VERSION=3.48.0 SQLITE_CFLAGS="-DSQLITE_OMIT_JSON -DSQLITE_MAX_VARIABLE_NUMBER=999" docker compose run --build --rm sqlite ``` -This is done to avoid compiling SQLite when you are not testing against a -specific version. +> [!NOTE] +> +> The `--build` argument is necessary if you've changed `SQLITE_CFLAGS` since +> the last run, as it's not part of the image tag. You can also rebuild the +> image separately by running `docker compose build sqlite`, optionally with +> `--no-cache` to ignore the cached build. + +In the future, the Django codebase may be more robust when tested against +different SQLite configurations and the `CFLAGS` workaround may no longer be +necessary. ### Other Versions diff --git a/compose.yml b/compose.yml index baa042d..c21b3e6 100644 --- a/compose.yml +++ b/compose.yml @@ -82,36 +82,6 @@ x-postgresql-base: &postgresql-base -c wal_level=minimal # 13+: -c wal_keep_size=0 -x-sqlite-lib-base: &sqlite-lib-base - image: django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION}-sqlite${SQLITE_VERSION} - pull_policy: never - build: - context: . - dockerfile_inline: | - FROM django-docker-box:${PYTHON_IMPLEMENTATION}-${PYTHON_VERSION} - SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] - # Only compile SQLite and set LD_PRELOAD if a version is specified. - RUN < Date: Tue, 28 Jan 2025 22:18:36 +0000 Subject: [PATCH 4/4] Fixed libsqlite3.so output path on SQLite 3.48.0+. --- compose.yml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compose.yml b/compose.yml index c21b3e6..301fb3e 100644 --- a/compose.yml +++ b/compose.yml @@ -305,7 +305,11 @@ services: cd /tmp/sqlite ./configure make - cp .libs/libsqlite3.so /tmp/ + if [ -f libsqlite3.so ]; then + cp libsqlite3.so /tmp/ + else + cp .libs/libsqlite3.so /tmp/ + fi rm -rf /tmp/sqlite fi EOF