Merge pull request #24 from di/updates

Normalize filenames, and other QOL updates

.python-version

@@ -1 +1 @@
-3.11.0
+3.11.8

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.11.0-slim-buster as build
+FROM python:3.11.8-slim-buster as build
 
 RUN set -x \
     && python3 -m venv /opt/conveyor
@@ -17,7 +17,7 @@ COPY requirements.txt /tmp/requirements.txt
 RUN set -x && pip --no-cache-dir --disable-pip-version-check install -r /tmp/requirements.txt
 
 
-FROM python:3.11.0-slim-buster
+FROM python:3.11.8-slim-buster
 
 ENV PYTHONUNBUFFERED 1
 ENV PYTHONPATH /opt/conveyor/src/

Makefile

@@ -0,0 +1,32 @@
+default:
+	@echo "Call a specific subcommand:"
+	@echo
+	@$(MAKE) -pRrq -f $(lastword $(MAKEFILE_LIST)) : 2>/dev/null\
+	| awk -v RS= -F: '/^# File/,/^# Finished Make data base/ {if ($$1 !~ "^[#.]") {print $$1}}'\
+	| sort\
+	| egrep -v -e '^[^[:alnum:]]' -e '^$@$$'
+	@echo
+	@exit 1
+
+# Optionally overriden by the user, if they're using a virtual environment manager.
+VENV ?= env
+
+# On Windows, venv scripts/shims are under `Scripts` instead of `bin`.
+VENV_BIN := $(VENV)/bin
+ifeq ($(OS),Windows_NT)
+	VENV_BIN := $(VENV)/Scripts
+endif
+
+$(VENV)/pyvenv.cfg:
+	# Create our Python 3 virtual environment
+	python3 -m venv $(VENV)
+	$(VENV_BIN)/python -m pip install --upgrade pip pip-tools tox
+
+.PHONY: tests
+tests: $(VENV)/pyvenv.cfg
+	. $(VENV_BIN)/activate && \
+		tox
+
+requirements.txt: requirements.in $(VENV)/pyvenv.cfg
+	. $(VENV_BIN)/activate && \
+		pip-compile --allow-unsafe --output-file=$@ $<

conveyor/views.py

@@ -18,6 +18,8 @@
 
 from aiohttp import web
 from botocore.config import Config as BotoCoreConfig
+from packaging.utils import parse_sdist_filename, parse_wheel_filename
+from packaging.utils import canonicalize_name, canonicalize_version
 
 ANON_CONFIG = BotoCoreConfig(signature_version=botocore.UNSIGNED)
 
@@ -30,6 +32,48 @@ async def not_found(request):
     return web.Response(status=404)
 
 
+async def _normalize_filename(filename):
+    if filename.endswith(".whl"):
+        name, ver, build, tags = parse_wheel_filename(filename)
+        return (
+            "-".join(
+                [
+                    canonicalize_name(name),
+                    canonicalize_version(ver),
+                ]
+                + (["".join(str(x) for x in build)] if build else [])
+                + [
+                    "-".join(str(x) for x in tags),
+                ]
+            )
+            + ".whl"
+        )
+    elif filename.endswith(".tar.gz"):
+        name, ver = parse_sdist_filename(filename)
+        return (
+            "-".join(
+                [
+                    canonicalize_name(name),
+                    canonicalize_version(ver),
+                ]
+            )
+            + ".tar.gz"
+        )
+    elif filename.endswith(".zip"):
+        name, ver = parse_sdist_filename(filename)
+        return (
+            "-".join(
+                [
+                    canonicalize_name(name),
+                    canonicalize_version(ver),
+                ]
+            )
+            + ".zip"
+        )
+    else:
+        return filename
+
+
 async def redirect(request):
     python_version = request.match_info["python_version"]
     project_l = request.match_info["project_l"]
@@ -38,8 +82,10 @@ async def redirect(request):
 
     # If the letter bucket doesn't match the first letter of the project, then
     # there is no point to going any further since it will be a 404 regardless.
-    if project_l != project_name[0]:
-        return web.Response(status=404, headers={'Reason': 'Incorrect project bucket'})
+    # Allow specifiying the exact first character of the actual filename (which
+    # might not be lowercase, to maintain backwards compatibility
+    if project_l != project_name[0].lower() and project_l != project_name[0]:
+        return web.Response(status=404, headers={"Reason": "Incorrect project bucket"})
 
     # If the filename we're looking for is a signature, then we'll need to turn
     # this into the *real* filename and a note that we're looking for the
@@ -72,8 +118,13 @@ async def redirect(request):
     # 302 redirect to that URL.
     for release in data.get("releases", {}).values():
         for file_ in release:
-            if (file_["filename"] == filename
-                    and file_["python_version"] == python_version):
+            if (
+                # Prefer that the normalized filename has been specified
+                _normalize_filename(file_["filename"]) == filename
+                # But also allow specifying the exact filename, to maintain
+                # backwards compatiblity
+                or file_["filename"] == filename
+            ) and file_["python_version"] == python_version:
                 # If we've found our filename, but we were actually looking for
                 # the *signature* of that file, then we need to check if it has
                 # a signature associated with it, and if so redirect to that,
@@ -88,7 +139,9 @@ async def redirect(request):
                             },
                         )
                     else:
-                        return web.Response(status=404, headers={'Reason': 'missing signature file'})
+                        return web.Response(
+                            status=404, headers={"Reason": "missing signature file"}
+                        )
                 # If we've found our filename, then we'll redirect to it.
                 else:
                     return web.Response(
@@ -101,7 +154,7 @@ async def redirect(request):
 
     # If we've gotten to this point, it means that we couldn't locate an url
     # to redirect to so we'll jsut 404.
-    return web.Response(status=404, headers={'Reason': 'no file found'})
+    return web.Response(status=404, headers={"Reason": "no file found"})
 
 
 async def fetch_key(s3, request, bucket, key):
@@ -117,17 +170,17 @@ async def index(request):
     session = request.app["boto.session"]()
     path = "index.html"
 
-    async with session.create_client('s3', config=ANON_CONFIG) as s3:
+    async with session.create_client("s3", config=ANON_CONFIG) as s3:
         try:
             key = await fetch_key(s3, request, bucket, path)
         except botocore.exceptions.ClientError:
             return web.Response(status=404)
 
         content_type, content_encoding = mimetypes.guess_type(path)
-        response = web.StreamResponse(status=200, reason='OK')
+        response = web.StreamResponse(status=200, reason="OK")
         response.content_type = content_type
         response.content_encoding = content_encoding
-        body = key['Body']
+        body = key["Body"]
         await response.prepare(request)
         while True:
             data = await body.read(4096)
@@ -157,7 +210,7 @@ async def documentation(request):
             status=302,
             headers={
                 "Location": location,
-            }
+            },
         )
 
     path = f"{project_name}/{path}"
@@ -167,7 +220,7 @@ async def documentation(request):
     bucket = request.app["settings"]["docs_bucket"]
     session = request.app["boto.session"]()
 
-    async with session.create_client('s3', config=ANON_CONFIG) as s3:
+    async with session.create_client("s3", config=ANON_CONFIG) as s3:
         try:
             key = await fetch_key(s3, request, bucket, path)
         except botocore.exceptions.ClientError:
@@ -179,10 +232,10 @@ async def documentation(request):
                 return web.HTTPMovedPermanently(location="/" + path + "/")
 
         content_type, content_encoding = mimetypes.guess_type(path)
-        response = web.StreamResponse(status=200, reason='OK')
+        response = web.StreamResponse(status=200, reason="OK")
         response.content_type = content_type
         response.content_encoding = content_encoding
-        body = key['Body']
+        body = key["Body"]
         await response.prepare(request)
         while True:
             data = await body.read(4096)

requirements.in

@@ -1,3 +1,4 @@
 aiobotocore
 aiohttp
 gunicorn
+packaging

requirements.txt

@@ -2,50 +2,47 @@
 # This file is autogenerated by pip-compile with Python 3.11
 # by the following command:
 #
-#    pip-compile --output-file=requirements.txt requirements.in
+#    pip-compile --allow-unsafe --output-file=requirements.txt requirements.in
 #
-aiobotocore==2.4.1
+aiobotocore==2.12.1
     # via -r requirements.in
-aiohttp==3.8.3
+aiohttp==3.9.3
     # via
     #   -r requirements.in
     #   aiobotocore
 aioitertools==0.11.0
     # via aiobotocore
 aiosignal==1.3.1
     # via aiohttp
-async-timeout==4.0.2
+attrs==23.2.0
     # via aiohttp
-attrs==22.1.0
-    # via aiohttp
-botocore==1.27.59
+botocore==1.34.51
     # via aiobotocore
-charset-normalizer==2.1.1
-    # via aiohttp
-frozenlist==1.3.3
+frozenlist==1.4.1
     # via
     #   aiohttp
     #   aiosignal
-gunicorn==20.1.0
+gunicorn==21.2.0
     # via -r requirements.in
-idna==3.4
+idna==3.6
     # via yarl
 jmespath==1.0.1
     # via botocore
-multidict==6.0.2
+multidict==6.0.5
     # via
     #   aiohttp
     #   yarl
-python-dateutil==2.8.2
+packaging==23.2
+    # via
+    #   -r requirements.in
+    #   gunicorn
+python-dateutil==2.9.0.post0
     # via botocore
 six==1.16.0
     # via python-dateutil
-urllib3==1.26.13
+urllib3==2.0.7
     # via botocore
-wrapt==1.14.1
+wrapt==1.16.0
     # via aiobotocore
-yarl==1.8.1
+yarl==1.9.4
     # via aiohttp
-
-# The following packages are considered to be unsafe in a requirements file:
-# setuptools

tests/test_views.py

@@ -0,0 +1,38 @@
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import pytest
+
+from conveyor.views import _normalize_filename
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize(
+    "filename, expected",
+    [
+        ("Flask-Common-0.2.0.tar.gz", "flask-common-0.2.tar.gz"),
+        ("websocket_client-0.52.0.tar.gz", "websocket-client-0.52.tar.gz"),
+        ("Sphinx-7.1.1.tar.gz", "sphinx-7.1.1.tar.gz"),
+        ("Foo_Bar-24.0.0.0.tar.gz", "foo-bar-24.tar.gz"),
+        ("Foo_Bar-24.0.0.0-py3-none-any.whl", "foo-bar-24-py3-none-any.whl"),
+        ("foo-24-py3-none-any.whl", "foo-24-py3-none-any.whl"),
+        (
+            "spam-1.0-420yolo-py3-none-any.whl",
+            "spam-1-420yolo-py3-none-any.whl",
+        ),  # Build tag
+        ("Foo_bar-24.0.0.0.zip", "foo-bar-24.zip"),
+    ],
+)
+async def test_normalize_filename(filename, expected):
+    result = await _normalize_filename(filename)
+
+    assert result == expected