From 9f8095a473e5a3da841dd655e7a91a17b47feb91 Mon Sep 17 00:00:00 2001 From: Nick Pope Date: Fri, 24 Nov 2023 13:36:43 +0000 Subject: [PATCH] [WIP] Updated everything! - Removed outdated configuration for Azure Pipelines - Removed outdated configuration for Travis CI - Removed obsolete `PIP_CACHE_VOLUME` from `.env` - Removed obsolete entries from `.gitignore` - Removed obsolete support for unofficial Oracle images - Removed obsolete `docker_push.sh` script - Added workflow for linting in GitHub Actions - Added `pre-commit`, `hadolint`, `ruff`, and `yamllint` for linting - Default `DJANGO_PATH` to `../django`, i.e. assume adjacent checkouts - Updated `Dockerfile` to use modern syntax, including: - Using here documents with `RUN` to avoid backslash escaping - Using `--mount=type=cache,...` with `RUN` to improve build speed - Updated `apt-get` and `pip` installation to be more optimal: - Enabled caching of downloads for both commands - Extracted packages to external files - Combined requirements files to optimise dependency resolution - Included `set -o errexit -o nounset -o pipefail -o xtrace` for `RUN` - Added forced color output, etc. to `Dockerfile` for use in CI - Tided up the location of injected and generated files in the image - Added preliminary support for alternative PyPy - Updated `docker-compose.yml` to use modern features: - Dropped the non-required `version` property - Used `additional_contexts` to source files from multiple places - Used `deploy: mode: global` for databases, etc. - Reverted to using `healthcheck` in place of `wait-for-it.sh` - Added configuration to speed up database services - Added support for Oracle Database Free 23c - Added GIS support for MariaDB and Oracle - Added Edge support for testing with Selenium - Renamed `Dockerfile` to `Containerfile` - Renamed `docker-compose.yml` to `compose.yaml` - Added support for running coverage via a custom entry point - Added support for extracting reports generated by lint/test tooling - Merged settings files into a single dynamic file - Updated README to document all of the changes and features - Added a LICENSE file using the same license as Django - Added a container for Redis to enable testing the Redis cache backend --- .azure.yml | 8 - .dockerignore | 4 +- .editorconfig | 8 +- .env | 9 +- .github/workflows/lint.yaml | 38 ++++ .gitignore | 2 - .hadolint.yaml | 5 + .pre-commit-config.yaml | 49 +++++ .travis.yml | 88 -------- Containerfile | 67 ++++++ Dockerfile | 37 ---- LICENSE | 27 +++ README.md | 99 ++++----- compose.yaml | 377 ++++++++++++++++++++++++++++++++++ coveragerc | 35 ++++ docker-compose.yml | 184 ----------------- docker_push.sh | 4 - entrypoint.bash | 12 ++ oracle/.gitkeep | 0 oracle_entrypoint.sh | 12 -- output/.gitignore | 2 + packages.txt | 18 ++ requirements.txt | 6 + settings.py | 69 +++++++ settings/__init__.py | 0 settings/test_mariadb.py | 50 ----- settings/test_mysql.py | 50 ----- settings/test_mysql_gis.py | 36 ---- settings/test_oracle.py | 52 ----- settings/test_postgres.py | 58 ------ settings/test_postgres_gis.py | 44 ---- settings/test_sqlite.py | 51 ----- settings/test_sqlite_gis.py | 36 ---- startup-mysql.sql | 1 + startup-oracle.sql | 3 + 35 files changed, 763 insertions(+), 778 deletions(-) delete mode 100644 .azure.yml create mode 100644 .github/workflows/lint.yaml delete mode 100644 .gitignore create mode 100644 .hadolint.yaml create mode 100644 .pre-commit-config.yaml delete mode 100644 .travis.yml create mode 100644 Containerfile delete mode 100644 Dockerfile create mode 100644 LICENSE create mode 100644 compose.yaml create mode 100644 coveragerc delete mode 100644 docker-compose.yml delete mode 100644 docker_push.sh create mode 100755 entrypoint.bash delete mode 100644 oracle/.gitkeep delete mode 100755 oracle_entrypoint.sh create mode 100644 output/.gitignore create mode 100644 packages.txt create mode 100644 requirements.txt create mode 100644 settings.py delete mode 100644 settings/__init__.py delete mode 100644 settings/test_mariadb.py delete mode 100644 settings/test_mysql.py delete mode 100644 settings/test_mysql_gis.py delete mode 100644 settings/test_oracle.py delete mode 100644 settings/test_postgres.py delete mode 100644 settings/test_postgres_gis.py delete mode 100644 settings/test_sqlite.py delete mode 100644 settings/test_sqlite_gis.py create mode 100644 startup-mysql.sql create mode 100644 startup-oracle.sql diff --git a/.azure.yml b/.azure.yml deleted file mode 100644 index b82fd74..0000000 --- a/.azure.yml +++ /dev/null @@ -1,8 +0,0 @@ -pool: - vmImage: 'vs2017-win2016' - -steps: -- script: | - docker pull microsoft/windowsservercore - docker-compose --version - displayName: Test diff --git a/.dockerignore b/.dockerignore index 9464c95..adc4c50 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,3 +1,3 @@ .git/ -.dockerignore/ -django \ No newline at end of file +.dockerignore +output/ diff --git a/.editorconfig b/.editorconfig index 37b4d92..491607a 100644 --- a/.editorconfig +++ b/.editorconfig @@ -1,12 +1,12 @@ root = true [*] -indent_style = space +charset = utf-8 +end_of_line = lf indent_size = 4 +indent_style = space insert_final_newline = true trim_trailing_whitespace = true -end_of_line = lf -charset = utf-8 -[*.yml] +[*.yaml] indent_size = 2 diff --git a/.env b/.env index 9467ae3..5a8c2c5 100644 --- a/.env +++ b/.env @@ -1,6 +1,7 @@ +PYTHON_IMPLEMENTATION=python PYTHON_VERSION=3.10 -POSTGRES_VERSION=13 -POSTGIS_VERSION=3.0 -MYSQL_VERSION=8 MARIADB_VERSION=10.5 -PIP_CACHE_VOLUME=pip-cache +MYSQL_VERSION=8 +ORACLE_VERSION=23.3.0.0 +POSTGIS_VERSION=3.0 +POSTGRES_VERSION=13 diff --git a/.github/workflows/lint.yaml b/.github/workflows/lint.yaml new file mode 100644 index 0000000..fbfb28c --- /dev/null +++ b/.github/workflows/lint.yaml @@ -0,0 +1,38 @@ +--- +name: Lint + +on: # yamllint disable-line rule:truthy + pull_request: + push: + branches: + - main + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +env: + COLUMNS: '120' + FORCE_COLOR: '1' + +jobs: + pre-commit: + runs-on: ubuntu-latest + name: pre-commit + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.13' + cache: pip + - name: Install packages + run: python -m pip install --upgrade pip pre-commit + - name: Run linting tools + run: pre-commit run --all-files + env: + PRE_COMMIT_COLOR: always diff --git a/.gitignore b/.gitignore deleted file mode 100644 index c3eed66..0000000 --- a/.gitignore +++ /dev/null @@ -1,2 +0,0 @@ -.idea/ -oracle/*.zip \ No newline at end of file diff --git a/.hadolint.yaml b/.hadolint.yaml new file mode 100644 index 0000000..1e55b63 --- /dev/null +++ b/.hadolint.yaml @@ -0,0 +1,5 @@ +--- +ignored: + - DL3013 # Don't complain when upgrading to latest version of pip + - DL3022 # Due to use of additional_contexts in compose.yaml + - DL3042 # Caching is desired with RUN --mount=type=cache diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..87b398f --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,49 @@ +--- +ci: + autoupdate_schedule: weekly +repos: + - repo: meta + hooks: + - id: check-hooks-apply + - id: check-useless-excludes + - repo: https://github.com/pre-commit/pre-commit-hooks + rev: v5.0.0 + hooks: + - id: check-added-large-files + - id: check-builtin-literals + - id: check-case-conflict + - id: check-docstring-first + - id: check-executables-have-shebangs + - id: check-merge-conflict + - id: check-shebang-scripts-are-executable + # - id: check-toml + - id: check-vcs-permalinks + - id: check-yaml + - id: debug-statements + - id: detect-private-key + - id: end-of-file-fixer + - id: file-contents-sorter + args: [--unique] + files: ^(?:packages|requirements)\.txt$ + - id: fix-byte-order-marker + - id: fix-encoding-pragma + args: [--remove] + - id: requirements-txt-fixer + - id: trailing-whitespace + - repo: https://github.com/astral-sh/ruff-pre-commit + rev: v0.7.0 + hooks: + - id: ruff + args: [--exit-non-zero-on-fix, --fix] + - id: ruff-format + args: [--check] + - repo: https://github.com/adrienverge/yamllint.git + rev: v1.35.1 + hooks: + - id: yamllint + args: [--strict] + - repo: https://github.com/hadolint/hadolint + rev: v2.12.0 + hooks: + - id: hadolint-docker + name: hadolint diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index f1d6e3e..0000000 --- a/.travis.yml +++ /dev/null @@ -1,88 +0,0 @@ -sudo: required -dist: xenial -services: -- docker -python: 3.10 -language: python -cache: pip - -env: -# Linting and docs -- COMPOSE_APP=flake8 -- COMPOSE_APP=docs - -# Quick Runs. Lowest supported database and Python version. -- COMPOSE_APP=sqlite PYTHON_VERSION=3.10 -- COMPOSE_APP=sqlite-gis PYTHON_VERSION=3.10 -- COMPOSE_APP=postgres PYTHON_VERSION=3.10 POSTGRES_VERSION=13 -- COMPOSE_APP=postgres-gis PYTHON_VERSION=3.10 POSTGRES_VERSION=13 POSTGIS_VERSION=3.0 -- COMPOSE_APP=mysql PYTHON_VERSION=3.10 MYSQL_VERSION=8 -- COMPOSE_APP=mysql-gis PYTHON_VERSION=3.10 MYSQL_VERSION=8 -- COMPOSE_APP=mariadb PYTHON_VERSION=3.10 MARIADB_VERSION=10.5 - -# Sqlite -- COMPOSE_APP=sqlite PYTHON_VERSION=3.10 - -# Sqlite gis: -- COMPOSE_APP=sqlite-gis PYTHON_VERSION=3.10 - -# Postgres: -- COMPOSE_APP=postgres PYTHON_VERSION=3.10 POSTGRES_VERSION=13 -- COMPOSE_APP=postgres PYTHON_VERSION=3.10 POSTGRES_VERSION=14 -- COMPOSE_APP=postgres PYTHON_VERSION=3.10 POSTGRES_VERSION=15 -- COMPOSE_APP=postgres PYTHON_VERSION=3.10 POSTGRES_VERSION=16 - -# Postgres gis: -- COMPOSE_APP=postgres-gis PYTHON_VERSION=3.10 POSTGRES_VERSION=13 POSTGIS_VERSION=3.0 -- COMPOSE_APP=postgres-gis PYTHON_VERSION=3.10 POSTGRES_VERSION=14 POSTGIS_VERSION=3.1 -- COMPOSE_APP=postgres-gis PYTHON_VERSION=3.10 POSTGRES_VERSION=15 POSTGIS_VERSION=3.3 -- COMPOSE_APP=postgres-gis PYTHON_VERSION=3.10 POSTGRES_VERSION=16 POSTGIS_VERSION=3.4 - -# MySQL: -- COMPOSE_APP=mysql PYTHON_VERSION=3.10 MYSQL_VERSION=8.0 - -# MySQL gis: -- COMPOSE_APP=mysql-gis PYTHON_VERSION=3.10 MYSQL_VERSION=8.0 - -# MariaDB: -- COMPOSE_APP=mariadb PYTHON_VERSION=3.10 MARIADB_VERSION=10.5 -- COMPOSE_APP=mariadb PYTHON_VERSION=3.10 MARIADB_VERSION=10.6 -- COMPOSE_APP=mariadb PYTHON_VERSION=3.10 MARIADB_VERSION=10.11 -- COMPOSE_APP=mariadb PYTHON_VERSION=3.10 MARIADB_VERSION=11.0 -- COMPOSE_APP=mariadb PYTHON_VERSION=3.10 MARIADB_VERSION=11.1 - -# Browsers: -- COMPOSE_APP=chrome PYTHON_VERSION=3.10 -- COMPOSE_APP=firefox PYTHON_VERSION=3.10 - -matrix: - allow_failures: - - env: COMPOSE_APP=firefox PYTHON_VERSION=3.10 - -before_install: - - echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee -a /etc/default/docker - - sudo service docker restart - -install: -- pip install docker-compose --upgrade - -script: -- git clone https://github.com/django/django.git --depth=1 /tmp/django -- export DJANGO_PATH=/tmp/django -- docker-compose pull --include-deps $COMPOSE_APP || true -- docker-compose build --pull $COMPOSE_APP -- | - if [[ $COMPOSE_APP == *-gis ]]; then - docker-compose run $COMPOSE_APP gis_tests - else - docker-compose run $COMPOSE_APP - fi - -after_failure: - - docker-compose logs - -deploy: - provider: script - script: bash docker_push.sh $COMPOSE_APP - on: - branch: main diff --git a/Containerfile b/Containerfile new file mode 100644 index 0000000..b9e0ea3 --- /dev/null +++ b/Containerfile @@ -0,0 +1,67 @@ +# syntax=docker/dockerfile:1.7 + +ARG PYTHON_IMPLEMENTATION +ARG PYTHON_VERSION +FROM ${PYTHON_IMPLEMENTATION}:${PYTHON_VERSION}-slim-bookworm + +LABEL org.opencontainers.image.authors="Django Software Foundation" +LABEL org.opencontainers.image.url="https://github.com/django/django-docker-box" +LABEL org.opencontainers.image.documentation="https://github.com/django/django-docker-box" +LABEL org.opencontainers.image.source="https://github.com/django/django-docker-box" +LABEL org.opencontainers.image.vendor="Django Software Foundation" +LABEL org.opencontainers.image.licenses="BSD-3-Clause" +LABEL org.opencontainers.image.title="Django Docker Box" +LABEL org.opencontainers.image.description="Container image for developing and testing Django." + +SHELL ["/bin/bash", "-o", "errexit", "-o", "nounset", "-o", "pipefail", "-o", "xtrace", "-c"] + +# Force colored output for various tooling in CI. +ENV COLUMNS=120 +ENV FORCE_COLOR=1 +ENV TERM="xterm-256color" + +ENV DEBIAN_FRONTEND=noninteractive +ENV PYTHONUNBUFFERED=1 + +# Create user and prepare directories. +RUN < /etc/apt/apt.conf.d/keep-cache + apt-get update --quiet --yes + xargs --arg-file=/django/packages.txt apt-get install --no-install-recommends --yes +EOF + +# Install all Python requirements in a single command. +COPY --chown=django:django requirements.txt /django/requirements/extra.txt +COPY --chown=django:django --from=src tests/requirements/ /django/requirements/ +COPY --chown=django:django --from=src docs/requirements.txt /django/requirements/docs.txt +RUN --mount=type=cache,target=/root/.cache/pip \ + --mount=type=cache,target=/root/.cache/uv <- + -c archive_mode=off + -c checkpoint_completion_target=0.9 + -c checkpoint_timeout=900 + -c fsync=off + -c full_page_writes=off + -c max_replication_slots=0 + -c max_wal_senders=0 + -c max_wal_size=4096 + -c synchronous_commit=off + -c wal_level=minimal + # 13+: -c wal_keep_size=0 + +x-memcached: &memcached-base + image: memcached:alpine + deploy: + mode: global + restart: unless-stopped + healthcheck: + test: echo stats | nc 127.0.0.1 11211 + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + start_interval: 1s + +x-redis: &redis-base + image: redis:alpine + deploy: + mode: global + restart: unless-stopped + healthcheck: + test: redis-cli ping | grep -i pong + interval: 5s + timeout: 5s + retries: 3 + start_period: 10s + start_interval: 1s + +x-cache-depends: &depends-on-caches + memcached-1: + condition: service_healthy + memcached-2: + condition: service_healthy + redis: + condition: service_healthy + +x-selenium-base: &selenium-base + shm_size: 2gb + environment: + - SE_EVENT_BUS_HOST=selenium-hub + - SE_EVENT_BUS_PUBLISH_PORT=4442 + - SE_EVENT_BUS_SUBSCRIBE_PORT=4443 + - SE_NODE_GRID_URL=http://selenium-hub:4444 + depends_on: + selenium-hub: + condition: service_started + deploy: + mode: global + restart: unless-stopped + +services: + + # Services: Databases + + mariadb-db: + <<: *mysql-base + image: &mariadb-image mariadb:${MARIADB_VERSION} + command: &mariadb-command >- + --innodb-fast-shutdown=3 + --innodb-flush-log-at-trx-commit=0 + --innodb-flush-method=nosync + --innodb-random-read-ahead + --skip-innodb-doublewrite + --skip-innodb-file-per-table + --skip-innodb-flush-sync + --skip-innodb-use-atomic-writes + --skip-name-resolve + healthcheck: &mariadb-healthcheck + <<: *mysql-base-healthcheck + test: healthcheck.sh --connect + + mariadb-gis-db: + <<: *mysql-base + image: *mariadb-image + command: *mariadb-command + healthcheck: *mariadb-healthcheck + + mysql-db: + <<: *mysql-base + image: &mysql-image mysql:${MYSQL_VERSION} + command: &mysql-command >- + --innodb-flush-log-at-trx-commit=0 + --innodb-flush-method=nosync + --innodb-random-read-ahead + --skip-innodb-doublewrite + --skip-innodb-extend-and-initialize + --skip-innodb-file-per-table + --skip-innodb-flush-sync + --skip-name-resolve + healthcheck: &mysql-healthcheck + <<: *mysql-base-healthcheck + test: mysqladmin ping --silent + + mysql-gis-db: + <<: *mysql-base + image: *mysql-image + command: *mysql-command + healthcheck: *mysql-healthcheck + + oracle-db: + <<: *oracle-base + image: container-registry.oracle.com/database/free:${ORACLE_VERSION} + + oracle-gis-db: + <<: *oracle-base + image: container-registry.oracle.com/database/free:${ORACLE_VERSION} + + postgresql-db: + <<: *postgresql-base + image: postgres:${POSTGRES_VERSION}-alpine + + postgresql-gis-db: + <<: *postgresql-base + image: postgis/postgis:${POSTGRES_VERSION}-${POSTGIS_VERSION} + + # Services: Caches + + memcached-1: + <<: *memcached-base + + memcached-2: + <<: *memcached-base + + redis: + <<: *redis-base + + # Services: Selenium + + selenium-chrome: + <<: *selenium-base + image: selenium/node-chrome + + selenium-edge: + <<: *selenium-base + image: selenium/node-edge + + selenium-firefox: + <<: *selenium-base + image: selenium/node-firefox + + selenium-hub: + image: selenium/hub + ports: + - "4442:4442" + - "4443:4443" + - "4444:4444" + deploy: + mode: global + restart: unless-stopped + healthcheck: + test: /opt/bin/check-grid.sh + interval: 15s + timeout: 30s + retries: 5 + + # Commands: Tests + + mariadb: + <<: *base + depends_on: + <<: *depends-on-caches + mariadb-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.db.backends.mysql + - DATABASE_HOST=mariadb-db + + mysql: + <<: *base + depends_on: + <<: *depends-on-caches + mysql-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.db.backends.mysql + - DATABASE_HOST=mysql-db + + oracle: + <<: *base + depends_on: + <<: *depends-on-caches + oracle-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.db.backends.oracle + - DATABASE_NAME=oracle-db:1521/freepdb1 + + postgresql: + <<: *base + depends_on: + <<: *depends-on-caches + postgresql-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.db.backends.postgresql + - DATABASE_HOST=postgresql-db + + sqlite: + <<: *base + depends_on: + <<: *depends-on-caches + environment: + - DATABASE_ENGINE=django.db.backends.sqlite3 + + # Commands: Tests: GIS + + mariadb-gis: + <<: *base + depends_on: + mariadb-gis-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.contrib.gis.db.backends.mysql + - DATABASE_HOST=mariadb-gis-db + + mysql-gis: + <<: *base + depends_on: + mysql-gis-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.contrib.gis.db.backends.mysql + - DATABASE_HOST=mysql-gis-db + + oracle-gis: + <<: *base + depends_on: + oracle-gis-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.contrib.gis.db.backends.oracle + - DATABASE_NAME=oracle-gis-db:1521/freepdb1 + + postgresql-gis: + <<: *base + depends_on: + postgresql-gis-db: + condition: service_healthy + environment: + - DATABASE_ENGINE=django.contrib.gis.db.backends.postgis + - DATABASE_HOST=postgresql-gis-db + + sqlite-gis: + <<: *base + environment: + - DATABASE_ENGINE=django.contrib.gis.db.backends.spatialite + + # Commands: Tests: Selenium + + chrome: + <<: *base + command: --selenium=chrome --selenium-hub=http://selenium-hub:4444/wd/hub + environment: + - DATABASE_ENGINE=django.db.backends.sqlite3 + depends_on: + selenium-hub: + condition: service_healthy + selenium-chrome: + condition: service_started + + edge: + <<: *base + command: --selenium=edge --selenium-hub=http://selenium-hub:4444/wd/hub + environment: + - DATABASE_ENGINE=django.db.backends.sqlite3 + depends_on: + selenium-hub: + condition: service_healthy + selenium-edge: + condition: service_started + + firefox: + <<: *base + command: --selenium=firefox --selenium-hub=http://selenium-hub:4444/wd/hub + environment: + - DATABASE_ENGINE=django.db.backends.sqlite3 + depends_on: + selenium-hub: + condition: service_healthy + selenium-firefox: + condition: service_started + + # Commands: Other + + docs: + <<: *base + entrypoint: ["make", "spelling"] + working_dir: /tests/django/docs + user: root + + pre-commit: + <<: *base + entrypoint: pre-commit run --all-files diff --git a/coveragerc b/coveragerc new file mode 100644 index 0000000..2c4863c --- /dev/null +++ b/coveragerc @@ -0,0 +1,35 @@ +[run] +branch = true +concurrency = multiprocessing,thread +data_file = /django/output/coverage/coverage.dat +dynamic_context = test_function +omit = + */django/utils/autoreload.py +parallel = true +relative_files = true +source = django + +[paths] +source = + /django/source + +[report] +ignore_errors = true +include_namespace_packages = true +omit = + */django/conf/locale/* + */tests/* +show_missing = true +skip_empty = true + +[html] +directory = /django/output/coverage/coverage.html +skip_empty = true +title = Django Test Coverage + +[json] +output = /django/output/coverage/coverage.json +show_contexts = true + +[xml] +output = /django/output/coverage/coverage.xml diff --git a/docker-compose.yml b/docker-compose.yml deleted file mode 100644 index 638f883..0000000 --- a/docker-compose.yml +++ /dev/null @@ -1,184 +0,0 @@ -version: "2.3" - -x-base: &base - image: djangobox/django-docker-box:${PYTHON_VERSION} - build: - context: ${DJANGO_PATH} - dockerfile: ${PWD}/Dockerfile - args: - - PYTHON_VERSION=${PYTHON_VERSION} - cache_from: - - djangobox/django-docker-box:${PYTHON_VERSION} - volumes: - - ${DJANGO_PATH}:/tests/django/ - - ./settings:/tests/django/tests/settings - - ./oracle_entrypoint.sh:/oracle_entrypoint.sh - # Using yaml merging here overwrites the arrays. Simpler to just mount this in every service. - - ./oracle:/oracle - entrypoint: bash -c "wait-for-it.sh -s -t 20 $${WAIT_FOR} -- python runtests.py $$@" -- - -x-postgres-base: &postgres-base - environment: - - POSTGRES_USER=django - - POSTGRES_PASSWORD=django - - POSTGRES_DB=django - # Set a few options to speed up postgresql tests. - command: -c fsync=off -c synchronous_commit=off -c full_page_writes=off - tmpfs: - - /var/lib/postgresql - -services: - sqlite: - <<: *base - environment: - - DJANGO_SETTINGS_MODULE=settings.test_sqlite - entrypoint: python runtests.py - depends_on: - - memcached - - memcached2 - - sqlite-gis: - <<: *base - entrypoint: python runtests.py - environment: - - DJANGO_SETTINGS_MODULE=settings.test_sqlite_gis - - postgres: - <<: *base - environment: - - DJANGO_SETTINGS_MODULE=settings.test_postgres - - WAIT_FOR=postgres-db:5432 - depends_on: - - postgres-db - - memcached - - memcached2 - postgres-db: - <<: *postgres-base - image: postgres:${POSTGRES_VERSION}-alpine - - postgres-gis: - <<: *base - environment: - - DJANGO_SETTINGS_MODULE=settings.test_postgres_gis - - WAIT_FOR=postgres-gis-db:5432 - depends_on: - - postgres-gis-db - postgres-gis-db: - <<: *postgres-base - image: postgis/postgis:${POSTGRES_VERSION}-${POSTGIS_VERSION} - - mysql: - <<: *base - environment: - - DJANGO_SETTINGS_MODULE=settings.test_mysql - - WAIT_FOR=mysql-db:3306 - depends_on: - - mysql-db - - memcached - - memcached2 - mysql-db: &mysql_base - image: mysql:${MYSQL_VERSION} - entrypoint: ['/entrypoint.sh', '--default-authentication-plugin=mysql_native_password'] - tmpfs: - - /var/lib/mysql - environment: - - MYSQL_USER=django - - MYSQL_PASSWORD=django - - MYSQL_DATABASE=django - - MYSQL_ALLOW_EMPTY_PASSWORD=1 - - mysql-gis: - <<: *base - environment: - - DJANGO_SETTINGS_MODULE=settings.test_mysql_gis - - WAIT_FOR=mysql-gis-db:3306 - depends_on: - - mysql-gis-db - mysql-gis-db: - <<: *mysql_base - volumes: - - /var/lib/mysql - - mariadb: - <<: *base - environment: - - DJANGO_SETTINGS_MODULE=settings.test_mariadb - - WAIT_FOR=mariadb-db:3306 - depends_on: - - mariadb-db - - memcached - - memcached2 - mariadb-db: - image: mariadb:${MARIADB_VERSION} - tmpfs: - - /var/lib/mysql - environment: - - MYSQL_USER=django - - MYSQL_PASSWORD=django - - MYSQL_DATABASE=django - - MYSQL_ALLOW_EMPTY_PASSWORD=1 - - oracle: - <<: *base - image: djangobox/django-docker-box:oracle-${PYTHON_VERSION} - environment: - - DJANGO_SETTINGS_MODULE=settings.test_oracle - - WAIT_FOR=oracle-db:1521 - entrypoint: /oracle_entrypoint.sh python runtests.py - depends_on: - - oracle-db - - memcached - - memcached2 - oracle-db: - image: sath89/oracle-12c - volumes: - - oracle:/u01/app/oracle/ - environment: - - WEB_CONSOLE=false - - memcached: &memcached - image: memcached:alpine - - memcached2: - <<: *memcached - - docs: - <<: *base - entrypoint: ["make", "spelling"] - working_dir: /tests/django/docs - user: root - - flake8: - <<: *base - entrypoint: flake8 - - # Browser tests - chrome: - <<: *base - command: --selenium=chrome --selenium-hub=http://chrome-browser:4444/wd/hub - environment: - - WAIT_FOR=chrome-browser:4444 - depends_on: - - chrome-browser - chrome-browser: - image: selenium/standalone-chrome - volumes: - - /dev/shm:/dev/shm - - # Firefox tests fail on main - firefox: - <<: *base - command: --selenium=firefox --selenium-hub=http://firefox-browser:4444/wd/hub - environment: - - WAIT_FOR=firefox-browser:4444 - depends_on: - - firefox-browser - firefox-browser: - image: selenium/standalone-firefox - volumes: - - /dev/shm:/dev/shm - - -volumes: - oracle: - diff --git a/docker_push.sh b/docker_push.sh deleted file mode 100644 index 915046e..0000000 --- a/docker_push.sh +++ /dev/null @@ -1,4 +0,0 @@ -#!/usr/bin/env bash - -echo "${DOCKER_PASSWORD}" | docker login -u "${DOCKER_USERNAME}" --password-stdin -docker-compose push $@ diff --git a/entrypoint.bash b/entrypoint.bash new file mode 100755 index 0000000..9d60a20 --- /dev/null +++ b/entrypoint.bash @@ -0,0 +1,12 @@ +#!/bin/bash + +if [[ -z "${COVERAGE}" ]] +then + python -Wall runtests.py ${@} +else + python -Wall -m coverage run runtests.py ${@} + python -m coverage combine + python -m coverage html + python -m coverage json + python -m coverage xml +fi diff --git a/oracle/.gitkeep b/oracle/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/oracle_entrypoint.sh b/oracle_entrypoint.sh deleted file mode 100755 index 18a333b..0000000 --- a/oracle_entrypoint.sh +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e -IFS=$'\n\t' - -if compgen -G "/oracle/*.zip" > /dev/null; then - sudo mkdir -p /opt/oracle - sudo unzip /oracle/*.zip -d /opt/oracle - sudo bash -c "echo /opt/oracle/instantclient* > /etc/ld.so.conf.d/oracle-instantclient.conf" - sudo ldconfig -fi -exec $@ diff --git a/output/.gitignore b/output/.gitignore new file mode 100644 index 0000000..d6b7ef3 --- /dev/null +++ b/output/.gitignore @@ -0,0 +1,2 @@ +* +!.gitignore diff --git a/packages.txt b/packages.txt new file mode 100644 index 0000000..34356df --- /dev/null +++ b/packages.txt @@ -0,0 +1,18 @@ +binutils +build-essential +default-libmysqlclient-dev +default-mysql-client +gdal-bin +gettext +git +libaio-dev +libenchant-2-dev +libgdal-dev +libgeoip-dev +libmemcached-dev +libpq-dev +libproj-dev +libsqlite3-mod-spatialite +pkg-config +rust-all +unzip diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f40f6fd --- /dev/null +++ b/requirements.txt @@ -0,0 +1,6 @@ +black +coverage +flake8 +flake8-isort +pre-commit +unittest-xml-reporting diff --git a/settings.py b/settings.py new file mode 100644 index 0000000..c9d1b08 --- /dev/null +++ b/settings.py @@ -0,0 +1,69 @@ +import os + + +def build_databases_setting(): + engine = os.environ["DATABASE_ENGINE"] + host = os.environ.get("DATABASE_HOST", "") + name = os.environ.get("DATABASE_NAME", "") + settings = {} + + for n, alias in enumerate(("default", "other"), start=1): + settings[alias] = entry = {"ENGINE": engine} + + if not engine.endswith((".sqlite", ".spatialite")): + entry |= { + "HOST": host, + "NAME": "django" if n < 2 else f"django{n}", + "USER": "django", + "PASSWORD": "django", + } + + if engine.endswith(".mysql"): + entry["TEST"] = {"CHARSET": "utf8"} + + if engine.endswith(".oracle"): + entry |= { + "NAME": name, + "TEST": { + "USER": f"{alias}_test", + "TBLSPACE": f"{alias}_test_tbls", + "TBLSPACE_TMP": f"{alias}_test_tbls_tmp", + }, + } + + return settings + + +DATABASES = build_databases_setting() + +SECRET_KEY = "django_tests_secret_key" + +PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"] + +CACHES = { + "default": { + "BACKEND": "django.core.cache.backends.locmem.LocMemCache", + }, + "pymemcache": { + "BACKEND": "django.core.cache.backends.memcached.PyMemcacheCache", + "LOCATION": "memcached-1:11211", + "KEY_PREFIX": "pymemcache:", + }, + "pylibmc": { + "BACKEND": "django.core.cache.backends.memcached.PyLibMCCache", + "LOCATION": "memcached-2:11211", + "KEY_PREFIX": "pylibmc:", + }, + "redis-py": { + "BACKEND": "django.core.cache.backends.redis.RedisCache", + "LOCATION": "redis://redis:6379", + "KEY_PREFIX": "redis:", + }, +} + +DEFAULT_AUTO_FIELD = "django.db.models.AutoField" +USE_TZ = False + +if os.environ.get("XUNIT", "0").lower() in {"1", "on", "true", "yes"}: + TEST_RUNNER = "xmlrunner.extra.djangotestrunner.XMLTestRunner" + TEST_OUTPUT_DIR = "/django/output/xunit" diff --git a/settings/__init__.py b/settings/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/settings/test_mariadb.py b/settings/test_mariadb.py deleted file mode 100644 index 82069ac..0000000 --- a/settings/test_mariadb.py +++ /dev/null @@ -1,50 +0,0 @@ -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'NAME': 'django', - 'HOST': 'mariadb-db', - 'TEST': { - 'CHARSET': 'utf8', - }, - }, - 'other': { - 'ENGINE': 'django.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'NAME': 'django2', - 'HOST': 'mariadb-db', - 'TEST': { - 'CHARSET': 'utf8', - }, - }, -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, - 'pymemcache': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': 'memcached:11211', - 'KEY_PREFIX': 'pymemcache:', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'memcached2:11211', - 'KEY_PREFIX': 'pylibmc:', - }, -} - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results/' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_mysql.py b/settings/test_mysql.py deleted file mode 100644 index b05ace0..0000000 --- a/settings/test_mysql.py +++ /dev/null @@ -1,50 +0,0 @@ -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'NAME': 'django', - 'HOST': 'mysql-db', - 'TEST': { - 'CHARSET': 'utf8', - }, - }, - 'other': { - 'ENGINE': 'django.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'NAME': 'django2', - 'HOST': 'mysql-db', - 'TEST': { - 'CHARSET': 'utf8', - }, - }, -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, - 'pymemcache': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': 'memcached:11211', - 'KEY_PREFIX': 'pymemcache:', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'memcached2:11211', - 'KEY_PREFIX': 'pylibmc:', - }, -} - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results/' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_mysql_gis.py b/settings/test_mysql_gis.py deleted file mode 100644 index 6185cd1..0000000 --- a/settings/test_mysql_gis.py +++ /dev/null @@ -1,36 +0,0 @@ -DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'NAME': 'django', - 'HOST': 'mysql-gis-db', - 'TEST': { - 'CHARSET': 'utf8', - }, - }, - 'other': { - 'ENGINE': 'django.contrib.gis.db.backends.mysql', - 'USER': 'root', - 'PASSWORD': '', - 'NAME': 'django2', - 'HOST': 'mysql-gis-db', - 'TEST': { - 'CHARSET': 'utf8', - }, - }, -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -GEOIP_PATH = '/geolite2/' - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results/' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_oracle.py b/settings/test_oracle.py deleted file mode 100644 index 99e4775..0000000 --- a/settings/test_oracle.py +++ /dev/null @@ -1,52 +0,0 @@ -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.oracle', - 'NAME': 'oracle-db:1521/xe', - 'USER': 'system', - 'PASSWORD': 'oracle', - 'TEST': { - 'USER': 'default_test', - 'TBLSPACE': 'default_test_tbls', - 'TBLSPACE_TMP': 'default_test_tbls_tmp', - }, - }, - 'other': { - 'ENGINE': 'django.db.backends.oracle', - 'NAME': 'oracle-db:1521/xe', - 'USER': 'system', - 'PASSWORD': 'oracle', - 'TEST': { - 'USER': 'other_test', - 'TBLSPACE': 'other_test_tbls', - 'TBLSPACE_TMP': 'other_test_tbls_tmp', - }, - }, -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, - 'pymemcache': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': 'memcached:11211', - 'KEY_PREFIX': 'pymemcache:', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'memcached2:11211', - 'KEY_PREFIX': 'pylibmc:', - }, -} - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results/' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_postgres.py b/settings/test_postgres.py deleted file mode 100644 index 85148ba..0000000 --- a/settings/test_postgres.py +++ /dev/null @@ -1,58 +0,0 @@ -# This is an example test settings file for use with the Django test suite. -# -# The 'sqlite3' backend requires only the ENGINE setting (an in- -# memory database will be used). All other backends will require a -# NAME and potentially authentication information. See the -# following section in the docs for more information: -# -# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/ -# -# The different databases that Django supports behave differently in certain -# situations, so it is recommended to run the test suite against as many -# database backends as possible. You may want to create a separate settings -# file for each of the backends you test against. - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql', - 'USER': 'django', - 'PASSWORD': 'django', - 'NAME': 'django', - 'HOST': 'postgres-db' - }, - 'other': { - 'ENGINE': 'django.db.backends.postgresql', - 'USER': 'django', - 'NAME': 'django2', - 'PASSWORD': 'django', - 'HOST': 'postgres-db' - } -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, - 'pymemcache': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': 'memcached:11211', - 'KEY_PREFIX': 'pymemcache:', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'memcached2:11211', - 'KEY_PREFIX': 'pylibmc:', - }, -} - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results/' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_postgres_gis.py b/settings/test_postgres_gis.py deleted file mode 100644 index 1e70561..0000000 --- a/settings/test_postgres_gis.py +++ /dev/null @@ -1,44 +0,0 @@ -# This is an example test settings file for use with the Django test suite. -# -# The 'sqlite3' backend requires only the ENGINE setting (an in- -# memory database will be used). All other backends will require a -# NAME and potentially authentication information. See the -# following section in the docs for more information: -# -# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/ -# -# The different databases that Django supports behave differently in certain -# situations, so it is recommended to run the test suite against as many -# database backends as possible. You may want to create a separate settings -# file for each of the backends you test against. - -DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'USER': 'django', - 'PASSWORD': 'django', - 'NAME': 'django', - 'HOST': 'postgres-gis-db' - }, - 'other': { - 'ENGINE': 'django.contrib.gis.db.backends.postgis', - 'USER': 'django', - 'NAME': 'django2', - 'PASSWORD': 'django', - 'HOST': 'postgres-gis-db' - } -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -GEOIP_PATH = '/geolite2/' - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_sqlite.py b/settings/test_sqlite.py deleted file mode 100644 index 1974bec..0000000 --- a/settings/test_sqlite.py +++ /dev/null @@ -1,51 +0,0 @@ -# This is an example test settings file for use with the Django test suite. -# -# The 'sqlite3' backend requires only the ENGINE setting (an in- -# memory database will be used). All other backends will require a -# NAME and potentially authentication information. See the -# following section in the docs for more information: -# -# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/ -# -# The different databases that Django supports behave differently in certain -# situations, so it is recommended to run the test suite against as many -# database backends as possible. You may want to create a separate settings -# file for each of the backends you test against. - -DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.sqlite3', - }, - 'other': { - 'ENGINE': 'django.db.backends.sqlite3', - } -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - - -CACHES = { - 'default': { - 'BACKEND': 'django.core.cache.backends.locmem.LocMemCache', - }, - 'pymemcache': { - 'BACKEND': 'django.core.cache.backends.memcached.PyMemcacheCache', - 'LOCATION': 'memcached:11211', - 'KEY_PREFIX': 'pymemcache:', - }, - 'pylibmc': { - 'BACKEND': 'django.core.cache.backends.memcached.PyLibMCCache', - 'LOCATION': 'memcached2:11211', - 'KEY_PREFIX': 'pylibmc:', - }, -} - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/settings/test_sqlite_gis.py b/settings/test_sqlite_gis.py deleted file mode 100644 index 55173c1..0000000 --- a/settings/test_sqlite_gis.py +++ /dev/null @@ -1,36 +0,0 @@ -# This is an example test settings file for use with the Django test suite. -# -# The 'sqlite3' backend requires only the ENGINE setting (an in- -# memory database will be used). All other backends will require a -# NAME and potentially authentication information. See the -# following section in the docs for more information: -# -# https://docs.djangoproject.com/en/dev/internals/contributing/writing-code/unit-tests/ -# -# The different databases that Django supports behave differently in certain -# situations, so it is recommended to run the test suite against as many -# database backends as possible. You may want to create a separate settings -# file for each of the backends you test against. - -DATABASES = { - 'default': { - 'ENGINE': 'django.contrib.gis.db.backends.spatialite', - }, - 'other': { - 'ENGINE': 'django.contrib.gis.db.backends.spatialite', - } -} - -SECRET_KEY = "django_tests_secret_key" - -# Use a fast hasher to speed up tests. -PASSWORD_HASHERS = [ - 'django.contrib.auth.hashers.MD5PasswordHasher', -] - -GEOIP_PATH = '/geolite2/' - -TEST_RUNNER = 'xmlrunner.extra.djangotestrunner.XMLTestRunner' -TEST_OUTPUT_DIR = '/tests/results/' -DEFAULT_AUTO_FIELD = 'django.db.models.AutoField' -USE_TZ = False diff --git a/startup-mysql.sql b/startup-mysql.sql new file mode 100644 index 0000000..4f947cf --- /dev/null +++ b/startup-mysql.sql @@ -0,0 +1 @@ +GRANT ALL PRIVILEGES ON *.* TO 'django'@'%' WITH GRANT OPTION; diff --git a/startup-oracle.sql b/startup-oracle.sql new file mode 100644 index 0000000..92ae6b1 --- /dev/null +++ b/startup-oracle.sql @@ -0,0 +1,3 @@ +ALTER SESSION SET CONTAINER=FREEPDB1; +CREATE USER django IDENTIFIED BY django; +GRANT DBA TO django;