Switch to main branch

.github/workflows/backend_checks.yml

@@ -0,0 +1,79 @@
+name: backend checks
+on:
+  push:
+    branches:
+      - main
+      - mwp_v1
+  pull_request:
+    paths:
+      - "backend/**"
+      - ".github/**"
+      - "docker/**"
+      - "tools/**"
+
+jobs:
+  backend-checks:
+    runs-on: self-hosted
+    env:
+      API_WORKERS: 1
+      RAY_CONFIG: "config_gpu.yaml"
+      JWT_SECRET: ${{ secrets.JWT_SECRET }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+      - name: Prepare environment
+        run: |
+          ./bin/setup-folders.sh
+          ./bin/setup-envs.sh --project_name action-runner --port_prefix 131
+          sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env
+          echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env
+          mkdir -p $HOME/spacy_models
+          mkdir -p $HOME/models_cache
+      - name: Build & Start Docker Containers
+        working-directory: docker
+        run: |
+          COMPOSE_PROFILES="weaviate,ray,background,backend" docker compose build
+          COMPOSE_PROFILES="weaviate,ray,background" docker compose up --wait --quiet-pull
+      - name: Check 1 - pytest runs without errors
+        working-directory: docker
+        run: |
+          docker compose run dats-backend-api /opt/envs/dats/bin/python -m pytest
+      - name: Check 2 - Database migrates without errors database
+        working-directory: docker
+        run: |
+          docker compose run -e PYTHONPATH='/dats_code/src' dats-backend-api /opt/envs/dats/bin/python migration/run_migrations.py
+      - name: Check 3 - Database schema is up-to-date after migration
+        working-directory: docker
+        run: |
+          docker compose run dats-backend-api /opt/envs/dats/bin/alembic check
+      - name: Start Remaining Docker Containers
+        working-directory: docker
+        run: |
+          COMPOSE_PROFILES="weaviate,ray,background,backend" docker compose up --wait --quiet-pull
+      - name: Check 4 - Test End-2-End importer script
+        working-directory: tools/importer
+        env:
+          TESTDATA_PASSWORD: ${{ secrets.TESTDATA_PASSWORD }}
+        run: |
+          pip install -r requirements.txt
+          wget -q http://ltdata1.informatik.uni-hamburg.de/dwts/totalitarismo.zip
+          unzip -q -P "$TESTDATA_PASSWORD" totalitarismo.zip
+          python dats_importer.py --input_dir json --backend_url http://localhost:13120/ --is_json --doctype text
+          python dats_importer.py --input_dir images --backend_url http://localhost:13120/ --doctype image
+      - name: Check 5 - pyright runs without errors
+        run: |
+          micromamba env create -f backend/environment.yml --yes
+          micromamba run -n dats pip install -r backend/src/app/preprocessing/ray_model_worker/requirements.txt
+          micromamba run -n dats pip install ray==2.32.0
+          micromamba run -n dats pyright
+      - name: Cleanup
+        working-directory: docker
+        if: always()
+        run: |
+          docker compose down -v --remove-orphans
+          micromamba env remove -n dats --yes
+          BACKEND_IMAGE=uhhlt/dats_backend:$(grep -oP 'DATS_BACKEND_DOCKER_VERSION=\K.*' .env)
+          RAY_IMAGE=uhhlt/dats_ray:$(grep -oP 'DATS_RAY_DOCKER_VERSION=\K.*' .env)
+          docker rmi $BACKEND_IMAGE $RAY_IMAGE

.github/workflows/frontend_checks.yml

@@ -0,0 +1,115 @@
+name: frontend checks
+on:
+  push:
+    branches:
+      - main
+      - mwp_v1
+  pull_request:
+    paths:
+      - ".github/**"
+      - "docker/**"
+      - "frontend/**"
+
+jobs:
+  frontend-checks:
+    runs-on: self-hosted
+    env:
+      API_WORKERS: 1
+      VITE_APP_SERVER: http://localhost:13120
+      COMPOSE_PROFILES: "weaviate,ray,background,backend,frontend"
+      RAY_CONFIG: "config_gpu.yaml"
+      JWT_SECRET: ${{ secrets.JWT_SECRET }}
+      BACKEND_HAS_NEW_REQUIREMENTS: false
+      RAY_HAS_NEW_REQUIREMENTS: false
+    steps:
+      - uses: actions/checkout@v3
+      - name: Checkout PR
+        if: github.event_name == 'pull_request'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: gh pr checkout ${{ github.event.pull_request.number }}
+      - name: Check for new backend requirements
+        id: check_backend_requirements
+        if: github.event_name == 'pull_request'
+        run: |
+          base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
+          if git diff --name-only $base_branch HEAD | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
+            echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
+          fi
+      - name: Check for new backend requirements on push
+        id: check_backend_requirements2
+        if: github.event_name == 'push'
+        run: |
+          if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
+            echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
+          fi
+      - name: Check for new ray requirements
+        id: check_ray_requirements
+        if: github.event_name == 'pull_request'
+        run: |
+          base_branch=$(git merge-base origin/${{ github.event.pull_request.base.ref }} HEAD)
+          if git diff --name-only $base_branch HEAD | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/requirements.txt'; then
+            echo "RAY_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
+          fi
+      - name: Check for new ray requirements on push
+        id: check_ray_requirements2
+        if: github.event_name == 'push'
+        run: |
+          if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/src/app/preprocessing/ray_model_worker/requirements.txt'; then
+            echo "RAY_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
+          fi
+      - name: Prepare environment
+        run: |
+          ./bin/setup-folders.sh
+          ./bin/setup-envs.sh --project_name action-runner --port_prefix 131
+          sed -i 's/\(DATS_FRONTEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          if [ "${{ env.RAY_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
+            sed -i 's/\(DATS_RAY_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          fi
+          if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
+            sed -i 's/\(DATS_BACKEND_DOCKER_VERSION=[0-9.]*\)/\1-${{ github.run_id }}/' docker/.env
+          fi
+          echo "SPACY_CACHE_DIR=$HOME/spacy_models" >> docker/.env
+          echo "MODELS_CACHE_DIR=$HOME/models_cache" >> docker/.env
+          mkdir -p $HOME/spacy_models
+          mkdir -p $HOME/models_cache
+      - name: Build & Start Docker Containers
+        working-directory: docker
+        run: |
+          if [ "${{ env.RAY_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
+            docker compose build ray
+          fi
+          if [ "${{ env.BACKEND_HAS_NEW_REQUIREMENTS }}" = "true" ]; then
+            docker compose build dats-backend-api
+          fi
+          docker compose build dats-frontend
+          docker compose up --wait --quiet-pull
+      - name: Setup node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 20
+          cache: npm
+          cache-dependency-path: frontend/package-lock.json
+      - name: Install dependencies
+        working-directory: frontend
+        run: npm ci
+      - name: Check 1 - api client is up-to-date
+        working-directory: frontend
+        run: |
+          npm run update-api
+          npm run generate-api
+      - name: Add and commit changes
+        uses: EndBug/add-and-commit@v9
+        if: github.event_name == 'pull_request'
+        with:
+          add: "frontend/src/api"
+          message: "Update OpenAPI spec and client"
+          pathspec_error_handling: exitImmediately
+          default_author: github_actions
+      - name: Cleanup
+        working-directory: docker
+        if: always()
+        run: |
+          docker compose down -v
+          FRONTEND_IMAGE=uhhlt/dats_frontend:$(grep -oP 'DATS_FRONTEND_DOCKER_VERSION=\K.*' .env)
+          docker rmi $FRONTEND_IMAGE

.github/workflows/release.yml

@@ -0,0 +1,106 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - "v*.*.*"
+
+jobs:
+  build_release:
+    name: build_release
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Get version from tag
+        id: extract_version
+        run: |
+          echo "version=${GITHUB_REF_NAME#v}" >> "$GITHUB_OUTPUT"
+        shell: bash
+
+      - name: Free Disk Space (Ubuntu)
+        uses: jlumbroso/free-disk-space@main
+        with:
+          tool-cache: false
+          docker-images: false
+          android: true
+          dotnet: true
+          haskell: true
+          large-packages: true
+          swap-storage: true
+
+      - name: Build and push backend container
+        uses: docker/build-push-action@v4
+        with:
+          context: ./backend
+          push: true
+          tags: |
+            "uhhlt/dats_backend:${{ steps.extract_version.outputs.version }}"
+      - name: Prepare environment
+        working-directory: frontend
+        run: cp .env.production.example .env.production
+      - name: Build and push frontend container
+        uses: docker/build-push-action@v4
+        with:
+          context: ./frontend
+          push: true
+          tags: |
+            "uhhlt/dats_frontend:${{ steps.extract_version.outputs.version }}"
+      - name: Build and push ray container
+        uses: docker/build-push-action@v4
+        with:
+          context: ./backend/src/app/preprocessing/ray_model_worker
+          push: true
+          tags: |
+            "uhhlt/dats_ray:${{ steps.extract_version.outputs.version }}"
+
+      - name: Create Release
+        uses: ncipollo/release-action@v1
+        with:
+          generateReleaseNotes: true
+  deploy_demo:
+    name: deploy_demo
+    needs: build_release
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        id: checkout
+        uses: actions/checkout@v3
+
+      - name: Write SSH config & key
+        id: write_ssh_keys
+        env:
+          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
+          SSH_CONFIG: ${{ secrets.SSH_CONFIG }}
+        run: |
+          install -m 600 -D /dev/null ~/.ssh/id_ed25519
+          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_ed25519
+          echo "$SSH_CONFIG" > ~/.ssh/config
+        shell: bash
+
+      - name: Add server to known hosts
+        env:
+          SSH_JUMP_HOST: ${{ secrets.SSH_JUMP_HOST }}
+        run: |
+          ssh-keyscan -H $SSH_JUMP_HOST >> ~/.ssh/known_hosts
+        shell: bash
+
+      - name: Deploy
+        id: deploy
+        env:
+          SSH_HOST: ${{ secrets.SSH_HOST }}
+        run: |
+          scp -o StrictHostKeyChecking=no bin/deploy.sh $SSH_HOST:~/deploy.sh
+          ssh -o ServerAliveInterval=60 -o ServerAliveCountMax=5 -o StrictHostKeyChecking=no $SSH_HOST './deploy.sh'
+          ssh -o StrictHostKeyChecking=no $SSH_HOST 'rm ./deploy.sh'
+        shell: bash

.pre-commit-config.yaml

@@ -0,0 +1,51 @@
+ci:
+  skip: [eslint, pyright]
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+      - id: check-yaml
+      - id: end-of-file-fixer
+      - id: requirements-txt-fixer
+      - id: trailing-whitespace
+  - repo: https://github.com/astral-sh/ruff-pre-commit
+    # Ruff version.
+    rev: v0.6.7
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [--fix]
+      # Run the formatter.
+      - id: ruff-format
+  - repo: local
+    hooks:
+      - id: pyright
+        name: "Pyright"
+        types: [python]
+        entry: bash -c 'ENV_NAME=dats source backend/_activate_current_env.sh && pyright $@'
+        language: system
+  - repo: https://github.com/pre-commit/mirrors-eslint
+    rev: v9.11.0
+    hooks:
+      - id: eslint
+        files: \.[jt]sx?$ # *.js, *.jsx, *.ts and *.tsx
+        types: [file]
+        args: [--max-warnings=0]
+        additional_dependencies:
+          - "@tanstack/[email protected]"
+          - [email protected]
+          - [email protected]
+          - [email protected]
+          - [email protected]
+  - repo: https://github.com/pre-commit/mirrors-prettier
+    rev: "v4.0.0-alpha.8"
+    hooks:
+      - id: prettier
+        files: \.([jt]sx?|json)$ # *.js, *.jsx, *.ts, *.tsx, *.json
+        args: [
+            --print-width,
+            "120",
+            # Without this, prettier will create a `node_modules` folder in the repository root
+            "--cache-location",
+            "frontend/node_modules/.cache/prettier",
+          ]

.vscode/extensions.json

@@ -0,0 +1,17 @@
+{
+  "recommendations": [
+    "ms-azuretools.vscode-docker",
+    // git
+    "eamodio.gitlens",
+    "github.vscode-github-actions",
+    "github.copilot",
+    // backend
+    "ms-python.python",
+    "ms-python.debugpy",
+    "ms-python.vscode-pylance",
+    "charliermarsh.ruff",
+    // frontend
+    "dbaeumer.vscode-eslint",
+    "esbenp.prettier-vscode"
+  ]
+}