fix document tag recommendation and annoscaling after disrupting vector DB code changes (#553)

* fix document tg recommendation and annoscaling after disrupting vector DB code changes

* [pre-commit.ci] auto fixes from pre-commit.com hooks

for more information, see https://pre-commit.ci

---------

Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com>

Author: fynnos

backend/src/app/core/annoscaling/annoscaling_service.py

@@ -1,20 +1,10 @@
 from time import perf_counter_ns
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    List,
-    Tuple,
-    TypeVar,
-)
+from typing import Any, Callable, Dict, Iterable, List, Tuple, TypeVar
 
 import numpy as np
 
 from app.core.data.crud.span_annotation import crud_span_anno
-from app.core.data.dto.search import (
-    SimSearchSentenceHit,
-)
+from app.core.data.dto.search import SimSearchSentenceHit
 from app.core.data.dto.span_annotation import SpanAnnotationCreate
 from app.core.data.orm.annotation_document import AnnotationDocumentORM
 from app.core.data.orm.source_document import SourceDocumentORM
@@ -142,7 +132,7 @@ def __suggest_similar_sentences(
                     client=client,
                     project_id=proj_id,
                     id=SentenceObjectIdentifier(sdoc_id=sdoc_id, sentence_id=sent_id),
-                    k=top_k,
+                    k=1,
                     threshold=0.0,
                 )
                 nearest.extend(

backend/src/app/core/ml/doc_tag_recommendation/doc_tag_recommendation_service.py

@@ -1,16 +1,6 @@
 import statistics
 from collections import defaultdict
-from typing import (
-    Any,
-    Callable,
-    Dict,
-    Iterable,
-    Iterator,
-    List,
-    Sequence,
-    Set,
-    TypeVar,
-)
+from typing import Any, Callable, Dict, Iterable, Iterator, List, Sequence, Set, TypeVar
 
 from app.core.data.crud.document_tag import crud_document_tag
 from app.core.data.crud.document_tag_recommendation import (
@@ -21,12 +11,12 @@
     DocumentTagRecommendationLinkCreate,
     DocumentTagRecommendationMethod,
 )
-from app.core.data.dto.search import (
-    SimSearchDocumentHit,
-)
+from app.core.data.dto.search import SimSearchDocumentHit
 from app.core.data.orm.document_tag import DocumentTagORM
 from app.core.db.sql_service import SQLService
 from app.core.vector.crud.document_embedding import crud_document_embedding
+from app.core.vector.dto.document_embedding import DocumentObjectIdentifier
+from app.core.vector.dto.search_results import SimSearchResult
 from app.core.vector.weaviate_service import WeaviateService
 from app.util.singleton_meta import SingletonMeta
 from weaviate import WeaviateClient
@@ -109,7 +99,7 @@ def classify_untagged_documents(
                         client, ml_job_id, project_id, sdoc_ids, sdocs_and_tags
                     )
 
-        dtos = self._deduplicate_document_classifications(dto_iter, multi_class)
+            dtos = self._deduplicate_document_classifications(dto_iter, multi_class)
 
         # Insert all generated tag recommendation DTOs into the database at once.
         crud_document_tag_recommendation_link.create_multi(db=db, create_dtos=dtos)
@@ -162,7 +152,7 @@ def __suggest_similar_documents(
                     client=client,
                     project_id=proj_id,
                     sdoc_id=sdoc_id,
-                    k=top_k,
+                    k=1,
                     threshold=0.0,
                 )
                 nearest.extend(
@@ -257,26 +247,25 @@ def _knn_suggestions(
             )
         sdoc_ids_to_classify = [sdoc.id for sdoc in sdocs_without_tags]
 
-        # TODO: Fix this
-        # nns = self.sim.knn_documents(project_id, sdoc_ids_to_classify, sdoc_ids, k=5)
-        nns = []
+        nns: List[List[SimSearchResult[DocumentObjectIdentifier]]] = []
 
         for sdoc_id in sdoc_ids_to_classify:
             # 1. Find k-nearest neighbors for the current sdoc_id
-            crud_document_embedding.search_near_sdoc(
+            result = crud_document_embedding.search_near_sdoc(
                 client=client,
                 project_id=project_id,
                 sdoc_id=sdoc_id,
                 k=5,
-                threshold=0.5,
+                threshold=0.0,
                 sdoc_ids=list(sdoc_ids),
             )
+            nns.append(result)
 
         for nn, sdoc in zip(nns, sdoc_ids_to_classify):
             pairs = [
                 (item.id, items.score)
                 for items in nn
-                for item in sdocs_and_tags[items.sdoc_id]
+                for item in sdocs_and_tags[items.id.sdoc_id]
             ]
             scores = defaultdict[int, list[float]](list)
             for id, score in pairs:

frontend/src/views/search/Statistics/SearchStatistics.tsx

@@ -83,7 +83,9 @@ function SearchStatistics({
           <Tabs value={tab} onChange={handleTabChange} variant="scrollable">
             <Tab label="Keywords" value="keywords" />
             <Tab label="Tags" value="tags" />
-            {projectCodes.data?.map((code) => <Tab key={code.id} label={code.name} value={`${code.id}`} />)}
+            {projectCodes.data?.map((code) => (
+              <Tab key={code.id} label={code.name} value={`${code.id}`} />
+            ))}
           </Tabs>
         </Stack>