changed conda setup to uv

.github/workflows/backend_checks.yml

@@ -14,7 +14,6 @@ jobs:
   backend-checks:
     runs-on: self-hosted
     env:
-      API_WORKERS: 1
       RAY_HAS_CHANGED: false
     steps:
       - uses: actions/checkout@v3
@@ -44,6 +43,7 @@ jobs:
           ./bin/setup-folders.sh
           ./bin/setup-envs.sh --project_name action-runner --port_prefix 131
           sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          sed -i 's/^RAY_DEVICE_IDS=.*/RAY_DEVICE_IDS=1/' docker/.env
           if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
             sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
             sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml
@@ -55,6 +55,7 @@ jobs:
       - name: Build & Start Docker Containers
         working-directory: docker
         run: |
+          source .env
           if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
             docker compose -f compose.ray.yml build
             docker compose -f compose.ray.yml up --wait
@@ -64,45 +65,53 @@ jobs:
       - name: Check 1 - pytest runs without errors
         working-directory: docker
         run: |
-          docker compose run dats-backend-api /opt/envs/dats/bin/python -m pytest
+          source .env
+          docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/python -m pytest
       - name: Check 2 - Database migrates without errors database
         working-directory: docker
         run: |
-          docker compose run -e PYTHONPATH='/dats_code/src' dats-backend-api /opt/envs/dats/bin/python migration/run_migrations.py
+          source .env
+          docker compose run --user $(id -u):$(id -g) -e PYTHONPATH='/dats_code/src' dats-backend-api /dats_code/.venv/bin/python migration/run_migrations.py
       - name: Check 3 - Database schema is up-to-date after migration
         working-directory: docker
         run: |
-          docker compose run dats-backend-api /opt/envs/dats/bin/alembic check
+          source .env
+          docker compose run --user $(id -u):$(id -g) dats-backend-api /dats_code/.venv/bin/alembic check
+      - name: Check 4 - install uv environment
+        working-directory: backend
+        run: |
+          uv sync
+      - name: Check 5 - pyright runs without errors
+        working-directory: backend
+        run: |
+          uv run pyright
       - name: Start Remaining Docker Containers
         working-directory: docker
         run: |
+          source .env
           COMPOSE_PROFILES="weaviate,background,backend" docker compose up --wait --quiet-pull
-      - name: Check 4 - Test End-2-End importer script
-        working-directory: tools/importer
+      - name: Check 6 - Test End-2-End importer script
+        working-directory: backend
         env:
           TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }}
         run: |
-          pip install -r requirements.txt
+          source .venv/bin/activate
+          cd ../tools/importer
+          uv pip install -r requirements.txt
           wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip
           unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip
           python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text
           python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image
-      - name: Check 5 - pyright runs without errors
-        run: |
-          micromamba env create -f backend/environment.yml --yes
-          micromamba run -n dats pip install -r backend/src/app/preprocessing/ray_model_worker/requirements.txt
-          micromamba run -n dats pip install ray==2.32.0
-          micromamba run -n dats pyright
       - name: Cleanup
         working-directory: docker
         if: always()
         run: |
+          source .env
           if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
             docker compose -f compose.ray.yml down -v
             RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env)
             docker rmi $RAY_IMAGE
           fi
           docker compose down -v --remove-orphans
-          micromamba env remove -n dats --yes
           BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)
           docker rmi $BACKEND_IMAGE

.github/workflows/frontend_checks.yml

@@ -64,6 +64,7 @@ jobs:
           ./bin/setup-folders.sh
           ./bin/setup-envs.sh --project_name action-runner --port_prefix 131
           sed -i 's/\(DATS_FRONTEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          sed -i 's/^RAY_DEVICE_IDS=.*/RAY_DEVICE_IDS=1/' docker/.env
           if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
             sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
           fi
@@ -78,6 +79,7 @@ jobs:
       - name: Build & Start Docker Containers
         working-directory: docker
         run: |
+          source .env
           if [ "${{ env.RAY_HAS_CHANGED }}" = "true" ]; then
             docker compose -f compose.ray.yml build
             docker compose -f compose.ray.yml up --wait

.github/workflows/ray_checks.yml

@@ -19,13 +19,15 @@ jobs:
           ./bin/setup-folders.sh
           ./bin/setup-envs.sh --project_name action-runner --port_prefix 131
           sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          sed -i 's/^RAY_DEVICE_IDS=.*/RAY_DEVICE_IDS=1/' docker/.env
           echo "RAY_CACHE_DIR=$HOME/ray_cache" >> docker/.env
           mkdir -p $HOME/ray_cache
           chmod 777 $HOME/ray_cache
           sed -i 's/ray_network/ray_network_${{ github.run_id }}/g' docker/compose.ray.yml
       - name: Build & Start Docker Containers
         working-directory: docker
         run: |
+          source .env
           docker compose -f compose.ray.yml build
           docker compose -f compose.ray.yml up --wait
       - name: Cleanup

.pre-commit-config.yaml

@@ -14,16 +14,16 @@ repos:
     hooks:
       # Run the linter.
       - id: ruff
-        args: [--fix]
+        args: [--fix, --config=backend/pyproject.toml]
       # Run the formatter.
       - id: ruff-format
   - repo: local
     hooks:
       - id: pyright
         name: "Pyright"
-        types: [python]
-        entry: ./bin/run-pyright.sh
-        language: script
+        entry: bash -c 'source backend/.venv/bin/activate && pyright "$@"' --
+        language: system
+        files: \.py$
   - repo: https://github.com/pre-commit/mirrors-eslint
     rev: v9.22.0
     hooks:

.vscode/launch.json

@@ -43,16 +43,16 @@
       "name": "celery",
       "type": "node-terminal",
       "request": "launch",
-      "command": "micromamba activate dats && celery -A app.celery.background_jobs.tasks worker -Q bgJobsQ,celery -P threads -l info -c 1 --without-gossip --without-mingle --without-heartbeat",
+      "command": "uv run celery -A app.celery.background_jobs.tasks worker -Q bgJobsQ,celery -P threads -l info -c 1 --without-gossip --without-mingle --without-heartbeat",
       "cwd": "${workspaceFolder}/backend/src",
       "envFile": "${workspaceFolder}/backend/.env"
     },
     {
       "name": "Pyright",
       "type": "node-terminal",
       "request": "launch",
-      "command": "micromamba activate dats && pyright",
-      "cwd": "${workspaceFolder}"
+      "command": "uv run pyright",
+      "cwd": "${workspaceFolder}/backend"
     },
     {
       "name": "Python: run_migrations.py",
@@ -70,15 +70,15 @@
       "name": "Alembic: check",
       "type": "node-terminal",
       "request": "launch",
-      "command": "micromamba activate dats && alembic check",
+      "command": "uv run alembic check",
       "cwd": "${workspaceFolder}/backend/src",
       "envFile": "${workspaceFolder}/backend/.env"
     },
     {
       "name": "Alembic: revision",
       "type": "node-terminal",
       "request": "launch",
-      "command": "micromamba activate dats && alembic revision --autogenerate -m \"vscode launcher\"",
+      "command": "uv run alembic revision --autogenerate -m \"vscode launcher\"",
       "cwd": "${workspaceFolder}/backend/src",
       "envFile": "${workspaceFolder}/backend/.env"
     }

.vscode/settings.json

@@ -1,15 +1,31 @@
 {
-  "python.envFile": "${workspaceFolder}/backend/.env",
-  "python.autoComplete.extraPaths": ["./backend/src"],
+  // general
   "editor.formatOnSave": true,
-  "prettier.configPath": "./frontend/package.json",
-  "javascript.preferences.importModuleSpecifierEnding": "js",
-  "typescript.preferences.importModuleSpecifierEnding": "js",
   "editor.codeActionsOnSave": {
     "source.organizeImports": "explicit"
   },
-  "python.analysis.diagnosticMode": "workspace",
   "files.insertFinalNewline": true,
+
+  // python / backend
+  "python.envFile": "${workspaceFolder}/backend/.env",
+  "python.autoComplete.extraPaths": ["./backend/src"],
+  "python.testing.pytestArgs": ["test"],
+  "python.testing.unittestEnabled": false,
+  "python.testing.pytestEnabled": true,
+  "python.testing.cwd": "backend/src",
+  "python.analysis.diagnosticMode": "workspace",
+  "python.analysis.extraPaths": ["./backend/src"],
+
+  // ruff (python linter)
+  "ruff.configuration": "backend/pyproject.toml",
+
+  // frontend
+  "javascript.preferences.importModuleSpecifierEnding": "js",
+  "typescript.preferences.importModuleSpecifierEnding": "js",
+
+  // formatting
+  "prettier.prettierPath": "./frontend/node_modules/prettier",
+  "prettier.configPath": "./frontend/package.json",
   "[yaml]": {
     "editor.tabSize": 2,
     "editor.insertSpaces": true
@@ -31,11 +47,5 @@
   },
   "[json]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
-  },
-  "python.testing.pytestArgs": ["test"],
-  "python.testing.unittestEnabled": false,
-  "python.testing.pytestEnabled": true,
-  "python.testing.cwd": "backend/src",
-  "prettier.prettierPath": "./frontend/node_modules/prettier",
-  "python.analysis.extraPaths": ["./backend/src"]
+  }
 }

backend/.dockerignore

@@ -22,3 +22,6 @@ notebooks
 Dockerfile
 Makefile
 README.md
+
+# igore venv
+.venv

backend/Dockerfile

@@ -2,67 +2,47 @@
 # docker push uhhlt/dats_backend:<version>
 
 FROM ubuntu:jammy-20221020 AS ubuntu
+# install uv
+COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/
 CMD ["/bin/bash"]
 
 # makes CUDA devices visible to the container by default
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=compute,utility
 
-# install basic tools to download files/models
-RUN apt-get update -q &&\
-    apt-get install -q -y --no-install-recommends bzip2 curl wget ca-certificates &&\
-    apt-get clean &&\
-    rm -rf /var/lib/apt/lists/* &&\
+# install basic Ubuntu packages including python3.11
+ARG DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Berlin
+RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone
+RUN apt-get update && apt-get install -y software-properties-common && add-apt-repository ppa:deadsnakes/ppa
+RUN apt-get update -q && \
+    apt-get install -q -y --no-install-recommends bzip2 curl wget ca-certificates libmagic1 libpq-dev build-essential python3.11 python3-pip ffmpeg && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* && \
     rm -r /var/cache
 
-# install micromamba for anaconda python package management
-ARG MICROMAMBA_VERSION=1.5.1
-RUN curl -Ls https://micro.mamba.pm/api/micromamba/linux-64/${MICROMAMBA_VERSION} -o mamba.tar.bz2
-RUN tar -xvj -f mamba.tar.bz2 bin/micromamba
-RUN rm mamba.tar.bz2
-
-# create the 'dats' python environment with all dependencies
-ENV MAMBA_ROOT_PREFIX=/opt
-COPY environment.yml .
-COPY requirements.txt /requirements.txt
-RUN micromamba create -f environment.yml -q -y &&\
-    micromamba clean -a -f -q -y &&\
-    find /opt/ -follow -type f -name '*.a' -delete &&\
-    find /opt/ -follow -type f -name '*.js.map' -delete &&\
-    rm -r /root/.cache
-
-
-# These settings & scripts are needed to automatically start any CMD in the container with the python environment
-ENV MAMBA_EXE="/bin/micromamba"
-ENV ENV_NAME=dats
-ARG MAMBA_DOCKERFILE_ACTIVATE=1
-COPY _entrypoint.sh /usr/local/bin/_entrypoint.sh
-COPY _activate_current_env.sh /usr/local/bin/_activate_current_env.sh
-COPY _dockerfile_shell.sh /usr/local/bin/_dockerfile_shell.sh
-ENTRYPOINT ["/usr/local/bin/_entrypoint.sh"]
-SHELL ["/usr/local/bin/_dockerfile_shell.sh"]
-
+# create the python environment
+ENV UV_LINK_MODE=copy
+ENV UV_COMPILE_BYTECODE=1
+ENV UV_LOCKED=1
 
+WORKDIR /dats_code
+RUN --mount=type=cache,target=/root/.cache/uv \
+    --mount=type=bind,source=uv.lock,target=uv.lock \
+    --mount=type=bind,source=pyproject.toml,target=pyproject.toml \
+    uv sync --directory . --no-dev --no-editable --no-install-project --no-install-workspace --no-managed-python
+ENV PATH="/dats_code/.venv/bin:${PATH}"
 
 # set up python env variables
-ARG DATS_ENV
-ENV DATS_ENV=${DATS_ENV} \
-    PYTHONFAULTHANDLER=1 \
-    PYTHONUNBUFFERED=1 \
-    PYTHONHASHSEED=random \
-    PYTHONDONTWRITEBYTECODE=1 \
-    CUPY_CACHE_IN_MEMORY=1
+ENV PYTHONFAULTHANDLER=1
+ENV PYTHONUNBUFFERED=1
+ENV PYTHONHASHSEED=random
+ENV PYTHONDONTWRITEBYTECODE=1
+ENV CUPY_CACHE_IN_MEMORY=1
 
 # allow running celery workers as superuser
 ENV C_FORCE_ROOT=1
 
-# For development, Jupyter remote kernel
-# Using inside the container:
-# jupyter lab --ip=0.0.0.0 --allow-root --NotebookApp.custom_display_url=http://127.0.0.1:8888
-ARG INSTALL_JUPYTER=false
-RUN bash -c "if [ $INSTALL_JUPYTER == 'true' ] ; then micromamba install jupyterlab -c conda-forge -q -y -n dats && micromamba clean -a -f -q -y ; fi"
-
 # copy the actual dats source code into the image
-WORKDIR /dats_code
 COPY . /dats_code
 WORKDIR /dats_code/src