Merge pull request #550 from uhh-lt/interactive-labeling

Interactive labeling

Author: bigabig

.github/workflows/frontend_checks.yml

@@ -33,14 +33,14 @@ jobs:
         run: |
           git fetch origin ${{ github.event.pull_request.base.ref }}
           base_branch="origin/${{ github.event.pull_request.base.ref }}"
-          if git diff --name-only $base_branch HEAD | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
+          if git diff --name-only $base_branch HEAD | grep -q -e '^backend/uv.lock'; then
             echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
           fi
       - name: Check for new backend requirements (push)
         id: check_backend_requirements_push
         if: github.event_name == 'push'
         run: |
-          if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/requirements.txt' -e '^backend/environment.yml'; then
+          if git diff --name-only ${{ github.event.before }} ${{ github.sha }} | grep -q -e '^backend/uv.lock'; then
             echo "BACKEND_HAS_NEW_REQUIREMENTS=true" >> $GITHUB_ENV
           fi
       - name: Check for changes in ray_model_worker (pull request)

.vscode/settings.json

@@ -49,5 +49,6 @@
   },
   "[json]": {
     "editor.defaultFormatter": "esbenp.prettier-vscode"
-  }
+  },
+  "python.analysis.exclude": ["**/node_modules", "**/__pycache__", ".git", "./backend/.venv", "./docker"]
 }

DATSIMPORT.md

@@ -17,6 +17,9 @@ JWT_SECRET=
 # Use `pwgen` to generate a secret or use any long random string
 SESSION_SECRET=
 
+# Use `python -c "import uuid; print(uuid.uuid4())"` to generate a UUID namespace for the backend
+UUID_NAMESPACE=
+
 # Where to store uploaded files.
 # <path_to_dats_repo>/docker/backend_repo
 SHARED_REPO_ROOT=/insert_path_to_dats_repo/docker/backend_repo
@@ -70,6 +73,7 @@ REDIS_PORT=13124
 REDIS_PASSWORD=dats123
 
 WEAVIATE_PORT=13132
+WEAVIATE_GRPC_PORT=13134
 
 RAY_HOST=localhost
 RAY_PORT=13130

backend/.env.example

@@ -30,6 +30,7 @@ dependencies = [
     "ffmpeg-python==0.2.0",
     "frozendict==2.3",
     "ftfy==6.1",
+    "hdbscan==0.8.40",
     "httpx==0.28.1",
     "isort==5.12.0",
     "itsdangerous==2.2.0",
@@ -56,7 +57,6 @@ dependencies = [
     "python-jose==3.3",
     "python-magic==0.4.27",
     "python-multipart==0.0.20",
-    "qdrant-client==1.9.1",
     "readability-lxml==0.8.1",
     "redis==4.3",
     "rope==1.9.0",
@@ -71,11 +71,10 @@ dependencies = [
     "tenacity==9.1.2",
     "tqdm==4.66.3",
     "twisted==22.10.0",
-    "typesense==0.21.0",
     "ujson>=5.10.0",
     "umap-learn==0.5.5",
     "uvicorn==0.23.2",
-    "weaviate-client==3.24.1",
+    "weaviate-client==4.14.4",
     "webdriver-manager==4.0.1",
     "yake==0.4.8",
 ]
@@ -92,6 +91,8 @@ dev = [
     "fasttext==0.9.3",
     "ffmpeg-python==0.2.0",
     "huggingface-hub==0.29.3",
+    "ipykernel==6.29.5",
+    "ipywidgets>=8.1.7",
     "lightning==2.5.1",
     "maverick-coref-de==1.0.5",
     "omegaconf==2.3.0",
@@ -101,11 +102,11 @@ dev = [
     "pytorch-crf==0.7.2",
     "quotect==1.1.5",
     "ray==2.44.1",
-    "sentence-transformers==4.0.1",
-    "setfit==1.1.1",
+    "sentence-transformers==4.1.0",
+    "setfit==1.1.2",
     "spacy==3.8.4",
     "spacy-curated-transformers==0.3.0",
-    "spacy-transformers==1.3.8",
+    "spacy-transformers==1.3.9",
     "timm==1.0.15",
     "torch==2.6.0",
     "torchaudio==2.6.0",

backend/pyproject.toml

@@ -0,0 +1,129 @@
+"""add perspectives tables (aspect, cluster, documentaspect, documentcluster)
+
+Revision ID: 114328a16f17
+Revises: 42c759c92f5b
+Create Date: 2025-07-03 14:29:46.966812
+
+"""
+
+from typing import Sequence, Union
+
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision: str = "114328a16f17"
+down_revision: Union[str, None] = "42c759c92f5b"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        "aspect",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("name", sa.String(), nullable=False),
+        sa.Column("doc_embedding_prompt", sa.Text(), nullable=False),
+        sa.Column("doc_modification_prompt", sa.Text(), nullable=True),
+        sa.Column("is_hierarchical", sa.Boolean(), nullable=False),
+        sa.Column("most_recent_job_id", sa.String(), nullable=True),
+        sa.Column(
+            "embedding_model", sa.String(), server_default="default", nullable=False
+        ),
+        sa.Column("project_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(["project_id"], ["project.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(op.f("ix_aspect_id"), "aspect", ["id"], unique=False)
+    op.create_index(
+        op.f("ix_aspect_project_id"), "aspect", ["project_id"], unique=False
+    )
+    op.create_table(
+        "cluster",
+        sa.Column("id", sa.Integer(), nullable=False),
+        sa.Column("is_outlier", sa.Boolean(), nullable=False),
+        sa.Column("name", sa.String(), nullable=True),
+        sa.Column("description", sa.Text(), nullable=True),
+        sa.Column("top_words", sa.ARRAY(sa.String()), nullable=True),
+        sa.Column("top_word_scores", sa.ARRAY(sa.Float()), nullable=True),
+        sa.Column("level", sa.Integer(), nullable=False),
+        sa.Column("top_docs", sa.ARRAY(sa.Integer()), nullable=True),
+        sa.Column("x", sa.Float(), nullable=True),
+        sa.Column("y", sa.Float(), nullable=True),
+        sa.Column("parent_cluster_id", sa.Integer(), nullable=True),
+        sa.Column("aspect_id", sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(["aspect_id"], ["aspect.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(
+            ["parent_cluster_id"], ["cluster.id"], ondelete="SET NULL"
+        ),
+        sa.PrimaryKeyConstraint("id"),
+    )
+    op.create_index(
+        op.f("ix_cluster_aspect_id"), "cluster", ["aspect_id"], unique=False
+    )
+    op.create_index(op.f("ix_cluster_id"), "cluster", ["id"], unique=False)
+    op.create_index(
+        op.f("ix_cluster_parent_cluster_id"),
+        "cluster",
+        ["parent_cluster_id"],
+        unique=False,
+    )
+    op.create_table(
+        "documentaspect",
+        sa.Column("sdoc_id", sa.Integer(), nullable=False),
+        sa.Column("aspect_id", sa.Integer(), nullable=False),
+        sa.Column("content", sa.Text(), nullable=False),
+        sa.Column("x", sa.Float(), nullable=True),
+        sa.Column("y", sa.Float(), nullable=True),
+        sa.ForeignKeyConstraint(["aspect_id"], ["aspect.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["sdoc_id"], ["sourcedocument.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("sdoc_id", "aspect_id"),
+    )
+    op.create_index(
+        op.f("ix_documentaspect_aspect_id"),
+        "documentaspect",
+        ["aspect_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_documentaspect_sdoc_id"), "documentaspect", ["sdoc_id"], unique=False
+    )
+    op.create_table(
+        "documentcluster",
+        sa.Column("sdoc_id", sa.Integer(), nullable=False),
+        sa.Column("cluster_id", sa.Integer(), nullable=False),
+        sa.Column("similarity", sa.Float(), nullable=True),
+        sa.Column("is_accepted", sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(["cluster_id"], ["cluster.id"], ondelete="CASCADE"),
+        sa.ForeignKeyConstraint(["sdoc_id"], ["sourcedocument.id"], ondelete="CASCADE"),
+        sa.PrimaryKeyConstraint("sdoc_id", "cluster_id"),
+    )
+    op.create_index(
+        op.f("ix_documentcluster_cluster_id"),
+        "documentcluster",
+        ["cluster_id"],
+        unique=False,
+    )
+    op.create_index(
+        op.f("ix_documentcluster_sdoc_id"), "documentcluster", ["sdoc_id"], unique=False
+    )
+    # ### end Alembic commands ###
+
+
+def downgrade() -> None:
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f("ix_documentcluster_sdoc_id"), table_name="documentcluster")
+    op.drop_index(op.f("ix_documentcluster_cluster_id"), table_name="documentcluster")
+    op.drop_table("documentcluster")
+    op.drop_index(op.f("ix_documentaspect_sdoc_id"), table_name="documentaspect")
+    op.drop_index(op.f("ix_documentaspect_aspect_id"), table_name="documentaspect")
+    op.drop_table("documentaspect")
+    op.drop_index(op.f("ix_cluster_parent_cluster_id"), table_name="cluster")
+    op.drop_index(op.f("ix_cluster_id"), table_name="cluster")
+    op.drop_index(op.f("ix_cluster_aspect_id"), table_name="cluster")
+    op.drop_table("cluster")
+    op.drop_index(op.f("ix_aspect_project_id"), table_name="aspect")
+    op.drop_index(op.f("ix_aspect_id"), table_name="aspect")
+    op.drop_table("aspect")
+    # ### end Alembic commands ###

backend/src/alembic/versions/114328a16f17_add_perspectives_tables.py

@@ -4,12 +4,14 @@
 from app.core.data.orm.user import UserORM
 from app.core.db.sql_service import SQLService
 from app.core.security import decode_jwt
+from app.core.vector.weaviate_service import WeaviateService
 from config import conf
 from fastapi import Depends, Query
 from fastapi.security import OAuth2PasswordBearer
 from jose import JWTError
 from pydantic import ValidationError
 from sqlalchemy.orm import Session
+from weaviate import WeaviateClient
 
 from api.util import credentials_exception
 
@@ -51,6 +53,15 @@ async def get_db_session() -> AsyncGenerator[Session, None]:
             session.close()
 
 
+async def get_weaviate_session() -> AsyncGenerator[WeaviateClient, None]:
+    session = WeaviateService().weaviate_session()
+    try:
+        yield session
+    finally:
+        if session is not None:
+            session.close()
+
+
 def get_current_user(
     db: Session = Depends(get_db_session), token: str = Depends(reusable_oauth2_scheme)
 ) -> UserORM:

backend/src/api/dependencies.py

@@ -6,8 +6,6 @@
     chat_session,
     retrieval_augmented_generation_with_session,
 )
-from app.core.search.filtering import Filter
-from app.core.search.sdoc_search.sdoc_search_columns import SdocColumns
 from fastapi import APIRouter, Depends
 from sqlalchemy.orm import Session
 
@@ -28,7 +26,7 @@ def rag_with_session(
     query: Union[str, List[str], int],
     top_k: int,
     threshold: float,
-    filter: Filter[SdocColumns],
+    sdoc_ids: Optional[List[int]],
     authz_user: AuthzUser = Depends(),
     session_id: Optional[str] = None,
     db: Session = Depends(get_db_session),
@@ -40,7 +38,7 @@ def rag_with_session(
         query=query,
         top_k=top_k,
         threshold=threshold,
-        filter=filter,
+        sdoc_ids=sdoc_ids,
         db=db,
         session_id=session_id,
     )