Merge branch 'main' into job-detail-leak

.github/dependabot.yml

@@ -19,3 +19,11 @@ updates:
   - dependency-name: lxml
     versions:
     - 4.6.2
+- package-ecosystem: github-actions
+  directory: "/"
+  groups:
+    github-actions:
+      patterns:
+        - "*"  # Group all Actions updates into a single larger pull request
+  schedule:
+    interval: daily

.github/workflows/ci.yml

@@ -5,7 +5,7 @@ jobs:
     runs-on: ubuntu-latest
     services:
       postgres:
-        image: postgres:10.1
+        image: postgres:15.3
         env:
           POSTGRES_USER: postgres
           POSTGRES_PASSWORD: postgres
@@ -16,12 +16,24 @@ jobs:
         options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
     steps:
       - name: Check out repository
-        uses: actions/checkout@v2
-      - uses: actions/setup-python@v2
+        uses: actions/checkout@v4
+      - name: Install platform dependencies
+        run: |
+          sudo apt -y update
+          sudo apt -y install --no-install-recommends \
+              texlive-latex-base \
+              texlive-latex-recommended \
+              texlive-plain-generic \
+              lmodern
+      - name: Install pandoc
+        run: |
+          wget https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-amd64.deb
+          sudo dpkg -i pandoc-2.17.1.1-1-amd64.deb
+      - uses: actions/setup-python@v5
         with:
           python-version: 3.9.16
       - name: Cache Python dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@v4
         env:
           cache-name: pythondotorg-cache-pip
         with:

.gitignore

@@ -10,6 +10,7 @@
 # $ git config --global core.excludesfile ~/.gitignore_global
 
 .sass-cache/
+docs/build
 media/*
 static-root/
 static/stylesheets/mq.css

.readthedocs.yaml

@@ -0,0 +1,15 @@
+# Read the Docs configuration file
+# See https://docs.readthedocs.io/en/stable/config-file/v2.html for details
+# Project page: https://readthedocs.org/projects/pythondotorg/
+
+version: 2
+
+build:
+  os: ubuntu-22.04
+  tools:
+    python: "3"
+
+  commands:
+      - python -m pip install -r docs-requirements.txt
+      - make -C docs html JOBS=$(nproc) BUILDDIR=_readthedocs
+      - mv docs/_readthedocs _readthedocs

Dockerfile

@@ -1,9 +1,48 @@
-FROM python:3.9-bullseye
+FROM python:3.9-bookworm
 ENV PYTHONUNBUFFERED=1
 ENV PYTHONDONTWRITEBYTECODE=1
+
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
+
+# Install System level build requirements, this is done before
+# everything else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        texlive-latex-base \
+        texlive-latex-recommended \
+        texlive-fonts-recommended \
+        texlive-plain-generic \
+        lmodern
+
+RUN case $(uname -m) in \
+         "x86_64")  ARCH=amd64 ;; \
+         "aarch64")  ARCH=arm64 ;; \
+    esac \
+    && wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \
+    && dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb
+
 RUN mkdir /code
 WORKDIR /code
+
 COPY dev-requirements.txt /code/
 COPY base-requirements.txt /code/
-RUN pip install -r dev-requirements.txt
+COPY prod-requirements.txt /code/
+COPY requirements.txt /code/
+
+RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    set -x \
+    && pip --disable-pip-version-check \
+        install \
+        -r dev-requirements.txt
+
 COPY . /code/

Dockerfile.cabotage

@@ -0,0 +1,49 @@
+FROM python:3.9-bullseye
+COPY --from=ewdurbin/nginx-static:1.25.x /usr/bin/nginx /usr/bin/nginx
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONDONTWRITEBYTECODE=1
+
+# By default, Docker has special steps to avoid keeping APT caches in the layers, which
+# is good, but in our case, we're going to mount a special cache volume (kept between
+# builds), so we WANT the cache to persist.
+RUN set -eux; \
+    rm -f /etc/apt/apt.conf.d/docker-clean; \
+    echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > /etc/apt/apt.conf.d/keep-cache;
+
+# Install System level build requirements, this is done before
+# everything else because these are rarely ever going to change.
+RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
+    --mount=type=cache,target=/var/lib/apt,sharing=locked \
+    set -x \
+    && apt-get update \
+    && apt-get install --no-install-recommends -y \
+        texlive-latex-base \
+        texlive-latex-recommended \
+        texlive-fonts-recommended \
+        texlive-plain-generic \
+        lmodern
+
+RUN case $(uname -m) in \
+         "x86_64")  ARCH=amd64 ;; \
+         "aarch64")  ARCH=arm64 ;; \
+    esac \
+    && wget --quiet https://github.com/jgm/pandoc/releases/download/2.17.1.1/pandoc-2.17.1.1-1-${ARCH}.deb \
+    && dpkg -i pandoc-2.17.1.1-1-${ARCH}.deb
+
+RUN mkdir /code
+WORKDIR /code
+
+COPY dev-requirements.txt /code/
+COPY base-requirements.txt /code/
+COPY prod-requirements.txt /code/
+COPY requirements.txt /code/
+
+RUN pip --no-cache-dir --disable-pip-version-check install --upgrade pip setuptools wheel
+
+RUN --mount=type=cache,target=/root/.cache/pip \
+    set -x \
+    && pip --disable-pip-version-check \
+        install \
+        -r requirements.txt -r prod-requirements.txt
+COPY . /code/
+RUN DJANGO_SETTINGS_MODULE=pydotorg.settings.static python manage.py collectstatic --noinput

Procfile

@@ -1,2 +1,4 @@
 release: python manage.py migrate --noinput
 web: bin/start-nginx gunicorn -c gunicorn.conf pydotorg.wsgi
+worker: celery -A pydotorg worker -l INFO
+worker-beat: celery -A pydotorg beat -l INFO --scheduler django_celery_beat.schedulers:DatabaseScheduler

README.md

@@ -1,6 +1,6 @@
 # python.org
 
-[![Build Status](https://travis-ci.org/python/pythondotorg.svg?branch=main)](https://travis-ci.org/python/pythondotorg)
+[![CI](https://github.com/python/pythondotorg/actions/workflows/ci.yml/badge.svg)](https://github.com/python/pythondotorg/actions/workflows/ci.yml)
 [![Documentation Status](https://readthedocs.org/projects/pythondotorg/badge/?version=latest)](https://pythondotorg.readthedocs.io/?badge=latest)
 
 ### General information

banners/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'banners.apps.BannersAppConfig'

base-requirements.txt

@@ -1,10 +1,10 @@
 dj-database-url==0.5.0
-django-pipeline==2.0.6
-django-sitetree==1.17.0
+django-pipeline==3.0.0  # 3.0.0 is first version that supports Django 4.2
+django-sitetree==1.18.0  # >=1.17.1 is (?) first version that supports Django 4.2
 django-apptemplates==1.5
 django-admin-interface==0.24.2
 django-translation-aliases==0.1.0
-Django==2.2.24
+Django==4.2.16
 docutils==0.12
 Markdown==3.3.4
 cmarkgfm==0.6.0
@@ -13,43 +13,44 @@ psycopg2-binary==2.8.6
 python3-openid==3.2.0
 python-decouple==3.4
 # lxml used by BeautifulSoup.
-lxml==4.6.3
+lxml==5.2.2
 cssselect==1.1.0
 feedparser==6.0.8
-beautifulsoup4==4.9.3
+beautifulsoup4==4.11.2
 icalendar==4.0.7
 chardet==4.0.0
+celery[redis]==5.3.6
+django-celery-beat==2.5.0
 # TODO: We may drop 'django-imagekit' completely.
-django-imagekit==4.0.2
-django-haystack==3.0
-elasticsearch>=5,<6
+django-imagekit==5.0  # 5.0 is first version that supports Django 4.2
+django-haystack==3.2.1
+elasticsearch>=7,<8
 # TODO: 0.14.0 only supports Django 1.8 and 1.11.
-django-tastypie==0.14.3
+django-tastypie==0.14.6  # 0.14.6 is first version that supports Django 4.2
 
 pytz==2021.1
 python-dateutil==2.8.2
 
 requests[security]>=2.26.0
 
-django-honeypot==1.0.1
-django-markupfield==2.0.0
-django-markupfield-helpers==0.1.1
+django-honeypot==1.0.4  # 1.0.4 is first version that supports Django 4.2
+django-markupfield==2.0.1
 
-django-allauth==0.41.0
+django-allauth==0.57.2 # 0.55.0 is first version that supports Django 4.2
 
 django-waffle==2.2.1
 
-djangorestframework==3.12.2
+djangorestframework==3.14.0  # 3.14.0 is first version that supports Django 4.1, 4.2 support hasnt been "released"
 django-filter==2.4.0
 django-ordered-model==3.4.3
 django-widget-tweaks==1.4.8
 django-countries==7.2.1
-xhtml2pdf==0.2.5
-django-easy-pdf3==0.1.2
 num2words==0.5.10
-django-polymorphic==3.0.0
+django-polymorphic==3.1.0  # 3.1.0 is first version that supports Django 4.0, unsure if it fully supports 4.2
 sorl-thumbnail==12.7.0
-docxtpl==0.12.0
-reportlab==3.6.6
 django-extensions==3.1.4
 django-import-export==2.7.1
+
+pypandoc==1.12
+panflute==2.3.0
+Unidecode==1.3.8

bin/start-nginx

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+psmgr=/tmp/nginx-buildpack-wait
+rm -f $psmgr
+mkfifo $psmgr
+
+n=1
+while getopts :f option ${@:1:2}
+do
+  case "${option}"
+  in
+    f) FORCE=$OPTIND; n=$((n+1));;
+  esac
+done
+
+# Initialize log directory.
+mkdir -p /tmp/logs/nginx
+touch /tmp/logs/nginx/access.log /tmp/logs/nginx/error.log
+echo 'buildpack=nginx at=logs-initialized'
+
+# Start log redirection.
+(
+  # Redirect nginx logs to stdout.
+  tail -qF -n 0 /tmp/logs/nginx/*.log
+  echo 'logs' >$psmgr
+) &
+
+# Start App Server
+(
+  # Take the command passed to this bin and start it.
+  # E.g. bin/start-nginx bundle exec unicorn -c config/unicorn.rb
+  COMMAND=${@:$n}
+  echo "buildpack=nginx at=start-app cmd=$COMMAND"
+  $COMMAND
+  echo 'app' >$psmgr
+) &
+
+if [[ -z "$FORCE" ]]
+then
+  FILE="/tmp/app-initialized"
+
+  # We block on app-initialized so that when nginx binds to $PORT
+  # are app is ready for traffic.
+  while [[ ! -f "$FILE" ]]
+  do
+    echo 'buildpack=nginx at=app-initialization'
+    sleep 1
+  done
+  echo 'buildpack=nginx at=app-initialized'
+fi
+
+# Start nginx
+(
+  # We expect nginx to run in foreground.
+  # We also expect a socket to be at /tmp/nginx.socket.
+  echo 'buildpack=nginx at=nginx-start'
+  cd /tmp
+  /usr/bin/nginx -p . -c /code/config/nginx.conf
+  echo 'nginx' >$psmgr
+) &
+
+# This read will block the process waiting on a msg to be put into the fifo.
+# If any of the processes defined above should exit,
+# a msg will be put into the fifo causing the read operation
+# to un-block. The process putting the msg into the fifo
+# will use it's process name as a msg so that we can print the offending
+# process to stdout.
+read exit_process <$psmgr
+echo "buildpack=nginx at=exit process=$exit_process"
+exit 1

blogs/__init__.py

@@ -1 +0,0 @@
-default_app_config = 'blogs.apps.BlogsAppConfig'

blogs/admin.py

@@ -10,11 +10,13 @@ class BlogEntryAdmin(admin.ModelAdmin):
     date_hierarchy = 'pub_date'
     actions = ['sync_new_entries']
 
+    @admin.action(
+        description="Sync new blog entries"
+    )
     def sync_new_entries(self, request, queryset):
         call_command('update_blogs')
         self.message_user(request, "Blog entries updated.")
 
-    sync_new_entries.short_description = "Sync new blog entries"
 
 
 @admin.register(FeedAggregate)

blogs/migrations/0003_alter_relatedblog_creator_and_more.py

@@ -0,0 +1,26 @@
+# Generated by Django 4.2.11 on 2024-09-05 17:10
+
+from django.conf import settings
+from django.db import migrations, models
+import django.db.models.deletion
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        migrations.swappable_dependency(settings.AUTH_USER_MODEL),
+        ('blogs', '0002_remove_translations_and_contributors'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='relatedblog',
+            name='creator',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_creator', to=settings.AUTH_USER_MODEL),
+        ),
+        migrations.AlterField(
+            model_name='relatedblog',
+            name='last_modified_by',
+            field=models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, related_name='%(app_label)s_%(class)s_modified', to=settings.AUTH_USER_MODEL),
+        ),
+    ]

blogs/parser.py

@@ -3,7 +3,7 @@
 
 from django.conf import settings
 from django.template.loader import render_to_string
-from django.utils.timezone import make_aware, utc
+from django.utils.timezone import make_aware
 
 from boxes.models import Box
 from .models import BlogEntry, Feed
@@ -16,7 +16,7 @@ def get_all_entries(feed_url):
 
     for e in d['entries']:
         published = make_aware(
-            datetime.datetime(*e['published_parsed'][:7]), timezone=utc
+            datetime.datetime(*e['published_parsed'][:7]), timezone=datetime.timezone.utc
         )
 
         entry = {