Map Upload WIP

- Add library to check file types
- Update docker and readme
- Update requirements
- Add field for incomplete uploads in case someone uploads a map, but never fills out metadata. These maps should be deleted if the user never finalizes the upload.
- Make a response for errors
- Fix the `is_text` check to account for `InMemoryUploads` being deleted when they are closed, meaning we couldn't use `with open(temp_file, "tr")` to check for text anymore.
- Make `ByteSized`. Might move into its own repo at some point.
- Fix binary fixture not reading as binary

Dockerfile

@@ -12,8 +12,10 @@ COPY requirements.txt /cncnet-map-api
 COPY requirements-dev.txt /cncnet-map-api
 COPY start.sh /cncnet-map-api
 
-RUN apt-get update && apt-get install -y liblzo2-dev
+RUN apt-get update && apt-get install -y liblzo2-dev # Compression library used by westwood.
+RUN apt-get install libmagic1 # File type checking.
 RUN pip install --upgrade pip
+# The cflags are needed to build the lzo library on Apple silicon.
 RUN CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r ./requirements-dev.txt
 
 RUN chmod +x /cncnet-map-api/start.sh

README.md

@@ -24,29 +24,34 @@ Just set up your environment file and run the full docker compose.
 
 ## Backend devs
 
+You can use the docker files if you'd like, but Django + docker is known to have issue attaching
+to debuggers and hitting breakpoints, so here are the native OS instructions.
+
 1. Download and install [pyenv](https://github.com/pyenv/pyenv)
 2. Install [PostgreSQL](https://www.postgresql.org/) for your system. This is required for Django
  - On Mac you can do `brew install postgresql` if you have brew installed.
-3. Checkout the repository
-4. Switch to the repository directory
-5. Setup Python
+3. Install LibMagic for [Python Magic](https://github.com/ahupp/python-magic)
+ - On Mac you can do `brew install libmagic` if you have breq installed.
+4. Checkout the repository
+5. Switch to the repository directory
+6. Setup Python
  - Install Python 3.12 `pyenv install 3.12` or whatever the latest python is.
  - Setup the virtual environments `pyenv virtualenv 3.12 cncnet-map-api`
  - Set the virtual enviornment for the directory `pyenv local cncnet-map-api`
-6. Setup requirements `pip install -r requirements-dev.txt`
+7. Setup requirements `pip install -r requirements-dev.txt`
  - On Apple Silicon you'll need to install lzo with `brew install lzo` then run
  `CFLAGS=-I$(brew --prefix)/include LDFLAGS=-L$(brew --prefix)/lib pip install -r requirements-dev.txt`
  to get `python-lzo` to install. You shouldn't need to include those flags again unless `python-lzo` updates.
-7. Install the pre-commit hooks `pre-commit install`
-8. Setup the environment variables
+8. Install the pre-commit hooks `pre-commit install`
+9. Setup the environment variables
  - Create a `.env` file at the root of the repo
  - Copy the contents of `example.env` to your `.env` file.
  - Fill out the required values in `.env`
  - If the app doesn't run due to a missing required variable, add said variable to `example.env` because the person
  who made the variable forgot to do so.
-9. Run the `db` service in `docker-compose`
-10. Load your `.env` file into your shell, (you can use `./load_env.sh`) then migrate the database `./manage.py migrate`
-11. `./manage.py runserver`
+10. Run the `db` service in `docker-compose`
+11. Load your `.env` file into your shell, (you can use `./load_env.sh`) then migrate the database `./manage.py migrate`
+12. `./manage.py runserver`
 
 I **strongly** recommend using PyCharm and the `.env` plugin for running the PyTests.
 

kirovy/migrations/0004_cncmap_incomplete_upload.py

@@ -0,0 +1,21 @@
+# Generated by Django 4.2.5 on 2024-04-13 19:28
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+ dependencies = [
+ ("kirovy", "0003_map_categories"),
+ ]
+
+ operations = [
+ migrations.AddField(
+ model_name="cncmap",
+ name="incomplete_upload",
+ field=models.BooleanField(
+ default=False,
+ help_text="If true, then the map file has been uploaded, but the map info has not been set yet.",
+ ),
+ ),
+ ]

kirovy/models/cnc_map.py

@@ -107,6 +107,11 @@ class CncMap(cnc_user.CncNetUserOwnedModel):
  )
  """:attr: Keep banned maps around so we can keep track of rule-breakers."""
 
+ incomplete_upload = models.BooleanField(
+ default=False,
+ help_text="If true, then the map file has been uploaded, but the map info has not been set yet.",
+ )
+
  def next_version_number(self) -> int:
  """Generate the next version to use for a map file.
 

kirovy/response.py

@@ -5,7 +5,7 @@
 class KirovyResponse(Response):
  def __init__(
  self,
- data: t.Optional[t.Union[t.ListResponseData, t.ResponseData]] = None,
+ data: t.Optional[t.BaseResponseData] = None,
  status: t.Optional[int] = None,
  template_name: t.Optional[str] = None,
  headers: t.Optional[t.DictStrAny] = None,

kirovy/services/cnc_gen_2_services.py

@@ -1,6 +1,7 @@
 import base64
 import logging
 
+import magic
 from PIL import Image
 import configparser
 import enum
@@ -140,12 +141,10 @@ def is_text(cls, uploaded_file: File) -> bool:
  :return:
  True if readable as text.
  """
- try:
- with uploaded_file.open("tr") as check_file:
- check_file.read()
- return True
- except UnicodeDecodeError:
- return False
+ magic_parser = magic.Magic(mime=True)
+ uploaded_file.seek(0)
+ mr_mime = magic_parser.from_buffer(uploaded_file.read())
+ return mr_mime == "text/plain"
 
  def extract_preview(self) -> t.Optional[Image.Image]:
  """Extract the map preview if it exists.

kirovy/typing/__init__.py

@@ -44,5 +44,9 @@ class ListResponseData(BaseResponseData):
  pagination_metadata: NotRequired[PaginationMetadata]
 
 
-class ResponseData(TypedDict):
+class ResponseData(BaseResponseData):
  result: DictStrAny
+
+
+class ErrorResponseData(BaseResponseData):
+ additional: NotRequired[DictStrAny]

kirovy/urls.py

@@ -14,13 +14,30 @@
  2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))
 """
 from django.contrib import admin
-from django.urls import path
+from django.urls import path, include
 
 from kirovy.views import test, cnc_map_views, permission_views
+from kirovy import typing as t
 
-urlpatterns = [
- path("admin/", admin.site.urls),
- path("test/jwt", test.TestJwt.as_view()),
- path("map-categories/", cnc_map_views.MapCategoryListCreateView.as_view()),
- path("ui-permissions/", permission_views.ListPermissionForAuthUser.as_view()),
+
+def _get_url_patterns() -> t.List[path]:
+ """Return the root level url patterns.
+
+ I added this because I wanted to have the root URLs at the top of the file,
+ but I didn't want to have other url files.
+ """
+ return [
+ path("admin/", admin.site.urls),
+ path("test/jwt", test.TestJwt.as_view()),
+ path("ui-permissions/", permission_views.ListPermissionForAuthUser.as_view()),
+ path("maps/", include(map_patterns)),
+ ]
+
+
+map_patterns = [
+ path("categories/", cnc_map_views.MapCategoryListCreateView.as_view()),
+ path("upload/<filename>/", cnc_map_views.MapFileUploadView.as_view()),
 ]
+
+
+urlpatterns = _get_url_patterns()

kirovy/utils/file_utils.py

@@ -1,5 +1,7 @@
+import collections
+import functools
 import hashlib
-from functools import partial
+from kirovy import typing as t
 
 from django.db.models.fields.files import FieldFile
 
@@ -21,3 +23,108 @@ def _hash_file(hasher: "_HASH", file: FieldFile, block_size: int) -> str:
  file.seek(0)
 
  return hasher.hexdigest()
+
+
+class ByteSized:
+ """A class to pretty format byte sizes, inspired by ``datetime.timedelta``'s functionality."""
+
+ _byte: int = 0
+ _kilo: int = 0
+ _mega: int = 0
+ _giga: int = 0
+ _tera: int = 0
+
+ def __new__(
+ cls,
+ byte: int = 0,
+ *,
+ kilo: int = 0,
+ mega: int = 0,
+ giga: int = 0,
+ tera: int = 0,
+ ) -> "ByteSized":
+
+ if any([x < 0 for x in [tera, giga, mega, kilo, byte]]):
+ raise AttributeError("Does not support args < 0")
+
+ self = object.__new__(cls)
+ b2k, br = divmod(byte, 1000)
+ self._byte = br
+
+ kilo = kilo + b2k
+ k2m, kr = divmod(kilo, 1000)
+ self._kilo = kr
+
+ mega = mega + k2m
+ m2g, mr = divmod(mega, 1000)
+ self._mega = mr
+
+ giga = giga + m2g
+ g2t, gr = divmod(giga, 1000)
+ self._giga = gr
+
+ self._tera = tera + g2t
+
+ return self
+
+ def __str__(self) -> str:
+ return ", ".join(
+ [f"{size}{desc}" for desc, size in self.__mapping.items() if size > 0]
+ )
+
+ @functools.cached_property
+ def __mapping(self) -> t.Dict[str, int]:
+ return collections.OrderedDict(
+ {
+ "TB": self.tera,
+ "GB": self.giga,
+ "MB": self.mega,
+ "KB": self.kilo,
+ "B": self.byte,
+ }
+ )
+
+ @property
+ def tera(self) -> int:
+ return self._tera
+
+ @property
+ def giga(self) -> int:
+ return self._giga
+
+ @property
+ def mega(self) -> int:
+ return self._mega
+
+ @property
+ def kilo(self) -> int:
+ return self._kilo
+
+ @property
+ def byte(self) -> int:
+ return self._byte
+
+ @functools.cached_property
+ def total_bytes(self) -> int:
+ total = 0
+ to_explode = [self._byte, self._kilo, self._mega, self._giga, self._tera]
+ for i, value in enumerate(to_explode):
+ exponent = 3 * i
+ magnitude = 10**exponent
+ total += value * magnitude
+ return total
+
+ def __gt__(self, other: "ByteSized") -> bool:
+ return self.total_bytes > other.total_bytes
+
+ def __lt__(self, other: "ByteSized") -> bool:
+ return self.total_bytes < other.total_bytes
+
+ def __ge__(self, other: "ByteSized") -> bool:
+ return self.total_bytes >= other.total_bytes
+
+ def __le__(self, other: "ByteSized") -> bool:
+ return self.total_bytes <= other.total_bytes
+
+ def __eq__(self, other: "ByteSized") -> bool:
+ return self.total_bytes == other.total_bytes

kirovy/views/cnc_map_views.py

@@ -1,15 +1,25 @@
+import logging
+
 from django.conf import settings
+from django.core.files.uploadedfile import UploadedFile
 from rest_framework import status
+from rest_framework.parsers import FileUploadParser, MultiPartParser
 from rest_framework.response import Response
 from rest_framework.views import APIView
 
-from kirovy import permissions
+from kirovy import permissions, typing as t, exceptions
 from kirovy.models import MapCategory
 from kirovy.request import KirovyRequest
+from kirovy.response import KirovyResponse
 from kirovy.serializers import cnc_map_serializers
+from kirovy.services.cnc_gen_2_services import CncGen2MapParser
+from kirovy.utils import file_utils
 from kirovy.views import base_views
 
 
+_LOGGER = logging.getLogger(__name__)
+
+
 class MapCategoryListCreateView(base_views.KirovyListCreateView):
  permission_classes = [permissions.IsAdmin | permissions.ReadOnly]
  serializer_class = cnc_map_serializers.MapCategorySerializer
@@ -28,3 +38,42 @@ class MapRetrieveUpdateView(base_views.KirovyRetrieveUpdateView):
 
 class MapDeleteView(base_views.KirovyDestroyView):
  ...
+
+
+class MapFileUploadView(APIView):
+ parser_classes = [MultiPartParser]
+ permission_classes = [permissions.CanUpload]
+
+ def post(
+ self, request: KirovyRequest, filename: str, format=None
+ ) -> KirovyResponse:
+
+ uploaded_file: UploadedFile = request.data["file"]
+ max_size = file_utils.ByteSized(mega=25)
+ uploaded_size = file_utils.ByteSized(uploaded_file.size)
+
+ if uploaded_size > max_size:
+ return KirovyResponse(
+ t.ErrorResponseData(
+ message="File too large",
+ additional={
+ "max_bytes": str(max_size),
+ "your_bytes": str(uploaded_file),
+ },
+ ),
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
+ try:
+ # TODO: Finish the map upload.
+ map_parser = CncGen2MapParser(uploaded_file)
+ except exceptions.InvalidMapFile as e:
+ return KirovyResponse(
+ t.ErrorResponseData(message="Invalid Map File"),
+ status=status.HTTP_400_BAD_REQUEST,
+ )
+
+ return KirovyResponse(
+ t.ResponseData(message="File uploaded successfully"),
+ status=status.HTTP_201_CREATED,
+ )

requirements-dev.txt

@@ -1,8 +1,7 @@
 -r requirements.txt
-pytest==7.*
+pytest==8.*
 pytest-mock==3.*
-black==22.*
-pre-commit==2.*
-pytest-django==4.5.2
-deptree==0.0.10
+black==24.*
+pre-commit==3.*
+pytest-django==4.*
 markdown>=3.4.4, <=4.0

requirements.txt

@@ -1,8 +1,9 @@
-django>=4.2.*, <=4.3 # 4.2 is the Long-Term Service version.
+django>=4.2.11, <=4.3 # 4.2 is the Long-Term Service version until 2026.
 psycopg2==2.*
 requests>=2.31.*, <3.0
 djangorestframework>=3.14.*, <4.0
 pyjwt[crypto]>=2.8.0
 pillow==10.*
 python-lzo==1.15
 ujson==5.*
+python-magic>=0.4.27