changed conda setup to uv

Author: floschne

.github/workflows/backend_checks.yml

@@ -64,35 +64,37 @@ jobs:
       - name: Check 1 - pytest runs without errors
         working-directory: docker
         run: |
-          docker compose run dats-backend-api /opt/envs/dats/bin/python -m pytest
+          docker compose run dats-backend-api /dats_code/.venv/bin/python -m pytest
       - name: Check 2 - Database migrates without errors database
         working-directory: docker
         run: |
-          docker compose run -e PYTHONPATH='/dats_code/src' dats-backend-api /opt/envs/dats/bin/python migration/run_migrations.py
+          docker compose run -e PYTHONPATH='/dats_code/src' dats-backend-api /dats_code/.venv/bin/python migration/run_migrations.py
       - name: Check 3 - Database schema is up-to-date after migration
         working-directory: docker
         run: |
-          docker compose run dats-backend-api /opt/envs/dats/bin/alembic check
+          docker compose run dats-backend-api /dats_code/.venv/bin/alembic check
       - name: Start Remaining Docker Containers
         working-directory: docker
         run: |
           COMPOSE_PROFILES="weaviate,background,backend" docker compose up --wait --quiet-pull
-      - name: Check 4 - Test End-2-End importer script
-        working-directory: tools/importer
+      - name: Check 4 - pyright runs without errors
+        working-directory: backend
+        run: |
+          uv sync --directory . --no-editable --no-install-project --no-install-workspace
+          source .venv/bin/activate
+          pyright
+      - name: Check 5 - Test End-2-End importer script
+        working-directory: backend
         env:
           TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }}
         run: |
-          pip install -r requirements.txt
+          source .venv/bin/activate
+          cd ../tools/importer
+          uv pip install -r requirements.txt
           wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip
           unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip
           python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text
           python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image
-      - name: Check 5 - pyright runs without errors
-        run: |
-          micromamba env create -f backend/environment.yml --yes
-          micromamba run -n dats pip install -r backend/src/app/preprocessing/ray_model_worker/requirements.txt
-          micromamba run -n dats pip install ray==2.32.0
-          micromamba run -n dats pyright
       - name: Cleanup
         working-directory: docker
         if: always()
@@ -103,6 +105,5 @@ jobs:
             docker rmi $RAY_IMAGE
           fi
           docker compose down -v --remove-orphans
-          micromamba env remove -n dats --yes
           BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)
           docker rmi $BACKEND_IMAGE

backend/.dockerignore

@@ -22,3 +22,6 @@ notebooks
 Dockerfile
 Makefile
 README.md
+
+# igore venv
+.venv

backend/Dockerfile

@@ -2,67 +2,47 @@
 # docker push uhhlt/dats_backend:<version>
 
 FROM ubuntu:jammy-20221020 AS ubuntu
+# install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 CMD ["/bin/bash"]
 
 # makes CUDA devices visible to the container by default
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 
-# install basic tools to download files/models
-RUN apt-get update -q &&\
-    apt-get install -q -y --no-install-recommends bzip2 curl wget ca-certificates &&\
-    apt-get clean &&\
-    rm -rf /var/lib/apt/lists/* &&\
+# install basic Ubuntu packages including python3.11
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Berlin
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update -q && \
+    apt-get install -q -y --no-install-recommends bzip2 curl wget ca-certificates libmagic1 libpq-dev build-essential python3.11 python3-pip ffmpeg && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     rm -r /var/cache
 
-# install micromamba for anaconda python package management
-ARG MICROMAMBA_VERSION=1.5.1
-RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/${MICROMAMBA_VERSION} -o mamba.tar.bz2
-RUN tar -xvj -f mamba.tar.bz2 bin/micromamba
-RUN rm mamba.tar.bz2
-
-# create the 'dats' python environment with all dependencies
-ENV MAMBA_ROOT_PREFIX=/opt
-COPY environment.yml .
-COPY requirements.txt /requirements.txt
-RUN micromamba create -f environment.yml -q -y &&\
-    micromamba clean -a -f -q -y &&\
-    find /opt/ -follow -type f -name '*.a' -delete &&\
-    find /opt/ -follow -type f -name '*.js.map' -delete &&\
-    rm -r /root/.cache
-
-
-# These settings & scripts are needed to automatically start any CMD in the container with the python environment
-ENV MAMBA_EXE="/bin/micromamba"
-ENV ENV_NAME=dats
-ARG MAMBA_DOCKERFILE_ACTIVATE=1
-COPY _entrypoint.sh /usr/local/bin/_entrypoint.sh
-COPY _activate_current_env.sh /usr/local/bin/_activate_current_env.sh
-COPY _dockerfile_shell.sh /usr/local/bin/_dockerfile_shell.sh
-ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"]
-SHELL ["/usr/local/bin/_dockerfile_shell.sh"]
-
+# create the python environment
+ENV UV_LINK_MODE=copy
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LOCKED=1
 
+WORKDIR /dats_code
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --directory . --no-dev --no-editable --no-install-project --no-install-workspace --no-managed-python
+ENV PATH="/dats_code/.venv/bin:${PATH}"
 
 # set up python env variables
-ARG DATS_ENV
-ENV DATS_ENV=${DATS_ENV} \
-    PYTHONFAULTHANDLER=1 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONHASHSEED=random \
-    PYTHONDONTWRITEBYTECODE=1 \
-    CUPY_CACHE_IN_MEMORY=1
+ENV PYTHONFAULTHANDLER=1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONHASHSEED=random
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV CUPY_CACHE_IN_MEMORY=1
 
 # allow running celery workers as superuser
 ENV C_FORCE_ROOT=1
 
-# For development, Jupyter remote kernel
-# Using inside the container:
-# jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
-ARG INSTALL_JUPYTER=false
-RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then micromamba install jupyterlab -c conda-forge -q -y -n dats && micromamba clean -a -f -q -y ; fi"
-
 # copy the actual dats source code into the image
-WORKDIR /dats_code
 COPY . /dats_code
 WORKDIR /dats_code/src