Remove patches, fix bugs

Author: gotsysdba

pyproject.toml

@@ -32,7 +32,7 @@ server = [
     "langchain-aimlapi==0.1.0",
     "langchain-cohere==0.4.6",
     "langchain-community==0.3.31",
-    "langchain-fireworks==0.3.0",
+    #"langchain-fireworks==0.3.0",
     "langchain-google-genai==2.1.12",
     "langchain-ibm==0.3.20",
     "langchain-mcp-adapters==0.1.13",
@@ -43,7 +43,7 @@ server = [
     "langchain-openai==0.3.35",
     "langchain-together==0.3.1",
     "langgraph==1.0.1",
-    "litellm==1.80.0",
+    "litellm==1.80.7",
     "llama-index==0.14.8",
     "lxml==6.0.2",
     "matplotlib==3.10.7",
@@ -70,6 +70,7 @@ test = [
     "pytest",
     "pytest-asyncio",
     "pytest-cov",
+    "types-jsonschema",
     "yamllint"
 ]
 

src/common/schema.py

@@ -346,18 +346,18 @@ class ChatRequest(LanguageModelParameters):
 #####################################################
 # Testbed
 #####################################################
-class TestSets(BaseModel):
-    """TestSets"""
+class QASets(BaseModel):
+    """QA Sets - Collection of Q&A test sets for testbed evaluation"""
 
     tid: str = Field(description="Test ID")
-    name: str = Field(description="Name of TestSet")
-    created: str = Field(description="Date TestSet Loaded")
+    name: str = Field(description="Name of QA Set")
+    created: str = Field(description="Date QA Set Loaded")
 
 
-class TestSetQA(BaseModel):
-    """TestSet Q&A"""
+class QASetData(BaseModel):
+    """QA Set Data - Question/Answer pairs for testbed evaluation"""
 
-    qa_data: list = Field(description="TestSet Q&A Data")
+    qa_data: list = Field(description="QA Set Data")
 
 
 class Evaluation(BaseModel):
@@ -390,6 +390,6 @@ class EvaluationReport(Evaluation):
 ModelEnabledType = ModelAccess.__annotations__["enabled"]
 OCIProfileType = OracleCloudSettings.__annotations__["auth_profile"]
 OCIResourceOCID = OracleResource.__annotations__["ocid"]
-TestSetsIdType = TestSets.__annotations__["tid"]
-TestSetsNameType = TestSets.__annotations__["name"]
-TestSetDateType = TestSets.__annotations__["created"]
+QASetsIdType = QASets.__annotations__["tid"]
+QASetsNameType = QASets.__annotations__["name"]
+QASetsDateType = QASets.__annotations__["created"]

src/launch_server.py

@@ -5,9 +5,6 @@
 # spell-checker:ignore configfile fastmcp noauth getpid procs litellm giskard ollama
 # spell-checker:ignore dotenv apiserver laddr
 
-# Patch litellm for Giskard/Ollama issue
-import server.patches.litellm_patch  # pylint: disable=unused-import, wrong-import-order
-
 # Set OS Environment before importing other modules
 # Set OS Environment (Don't move their position to reflect on imports)
 # pylint: disable=wrong-import-position

src/server/api/utils/databases.py

@@ -100,12 +100,14 @@ def _test(config: Database) -> None:
     except oracledb.DatabaseError:
         logger.info("Refreshing %s database connection.", config.name)
         _ = connect(config)
-    except ValueError as ex:
-        raise DbException(status_code=400, detail=f"Database: {str(ex)}") from ex
+    except DbException:
+        raise
     except PermissionError as ex:
         raise DbException(status_code=401, detail=f"Database: {str(ex)}") from ex
     except ConnectionError as ex:
         raise DbException(status_code=503, detail=f"Database: {str(ex)}") from ex
+    except ValueError as ex:
+        raise DbException(status_code=400, detail=f"Database: {str(ex)}") from ex
     except Exception as ex:
         raise DbException(status_code=500, detail=str(ex)) from ex
 
@@ -136,7 +138,7 @@ def connect(config: Database) -> oracledb.Connection:
     include_fields = set(DatabaseAuth.model_fields.keys())
     db_authn = config.model_dump(include=include_fields)
     if any(not db_authn[key] for key in ("user", "password", "dsn")):
-        raise ValueError("missing connection details")
+        raise DbException(status_code=400, detail=f"Database: {config.name} missing connection details.")
 
     logger.info("Connecting to Database: %s", config.dsn)
     # If a wallet password is provided but no wallet location is set
@@ -249,7 +251,7 @@ def get_databases(
         for db in databases:
             try:
                 db_conn = connect(config=db)
-            except (ValueError, PermissionError, ConnectionError, LookupError):
+            except (DbException, PermissionError, ConnectionError, LookupError):
                 continue
             db.vector_stores = _get_vs(db_conn)
             db.connected = True

src/server/api/utils/oci.py

@@ -185,6 +185,9 @@ def get_namespace(config: OracleCloudSettings) -> str:
         client = init_client(client_type, config)
         config.namespace = client.get_namespace().data
         logger.info("OCI: Namespace = %s", config.namespace)
+    except OciException:
+        # Re-raise OciException from init_client without wrapping
+        raise
     except oci.exceptions.InvalidConfig as ex:
         raise OciException(status_code=400, detail="Invalid Config") from ex
     except oci.exceptions.ServiceError as ex:

src/server/api/utils/testbed.py

@@ -95,25 +95,25 @@ def get_testsets(db_conn: Connection) -> list:
     sql = "SELECT tid, name, to_char(created) FROM oai_testsets ORDER BY created"
     results = utils_databases.execute_sql(db_conn, sql)
     try:
-        testsets = [schema.TestSets(tid=tid.hex(), name=name, created=created) for tid, name, created in results]
+        testsets = [schema.QASets(tid=tid.hex(), name=name, created=created) for tid, name, created in results]
     except TypeError:
         create_testset_objects(db_conn)
 
     return testsets
 
 
-def get_testset_qa(db_conn: Connection, tid: schema.TestSetsIdType) -> schema.TestSetQA:
+def get_testset_qa(db_conn: Connection, tid: schema.QASetsIdType) -> schema.QASetData:
     """Get list of TestSet Q&A"""
     logger.info("Getting TestSet Q&A for TID: %s", tid)
     binds = {"tid": tid}
     sql = "SELECT qa_data FROM oai_testset_qa where tid=:tid"
     results = utils_databases.execute_sql(db_conn, sql, binds)
     qa_data = [qa_data[0] for qa_data in results]
 
-    return schema.TestSetQA(qa_data=qa_data)
+    return schema.QASetData(qa_data=qa_data)
 
 
-def get_evaluations(db_conn: Connection, tid: schema.TestSetsIdType) -> list[schema.Evaluation]:
+def get_evaluations(db_conn: Connection, tid: schema.QASetsIdType) -> list[schema.Evaluation]:
     """Get list of Evaluations for a TID"""
     logger.info("Getting Evaluations for: %s", tid)
     evaluations = []
@@ -133,7 +133,7 @@ def get_evaluations(db_conn: Connection, tid: schema.TestSetsIdType) -> list[sch
 
 def delete_qa(
     db_conn: Connection,
-    tid: schema.TestSetsIdType,
+    tid: schema.QASetsIdType,
 ) -> None:
     """Delete Q&A"""
     binds = {"tid": tid}
@@ -144,11 +144,11 @@ def delete_qa(
 
 def upsert_qa(
     db_conn: Connection,
-    name: schema.TestSetsNameType,
-    created: schema.TestSetDateType,
+    name: schema.QASetsNameType,
+    created: schema.QASetsDateType,
     json_data: json,
-    tid: schema.TestSetsIdType = None,
-) -> schema.TestSetsIdType:
+    tid: schema.QASetsIdType = None,
+) -> schema.QASetsIdType:
     """Upsert Q&A"""
     logger.info("Upsert TestSet: %s - %s", name, created)
     parsed_data = json.loads(json_data)
@@ -270,7 +270,7 @@ def build_knowledge_base(
     return testset
 
 
-def process_report(db_conn: Connection, eid: schema.TestSetsIdType) -> schema.EvaluationReport:
+def process_report(db_conn: Connection, eid: schema.QASetsIdType) -> schema.EvaluationReport:
     """Process an evaluate report"""
 
     # Main

src/server/api/v1/databases.py

@@ -76,24 +76,25 @@ async def databases_update(
 
     db.connected = False
     try:
-        payload.config_dir = db.config_dir
-        payload.wallet_location = db.wallet_location
-        logger.debug("Testing Payload: %s", payload)
-        db_conn = utils_databases.connect(payload)
-    except (ValueError, PermissionError, ConnectionError, LookupError) as ex:
+        # Create a test config with payload values to test connection
+        # Only update the actual db object after successful connection
+        test_config = db.model_copy(update=payload.model_dump(exclude_unset=True))
+        logger.debug("Testing Database: %s", test_config)
+        db_conn = utils_databases.connect(test_config)
+    except utils_databases.DbException as ex:
+        raise HTTPException(status_code=ex.status_code, detail=ex.detail) from ex
+    except (PermissionError, ConnectionError, LookupError) as ex:
         status_code = 500
-        if isinstance(ex, ValueError):
-            status_code = 400
-        elif isinstance(ex, PermissionError):
+        if isinstance(ex, PermissionError):
             status_code = 401
         elif isinstance(ex, LookupError):
             status_code = 404
         elif isinstance(ex, ConnectionError):
             status_code = 503
-        else:
-            raise
         raise HTTPException(status_code=status_code, detail=f"Database: {db.name} {ex}.") from ex
-    for key, value in payload.model_dump().items():
+
+    # Connection successful - now update the actual db object
+    for key, value in payload.model_dump(exclude_unset=True).items():
         setattr(db, key, value)
 
     # Manage Connections; Unset and disconnect other databases

src/server/api/v1/testbed.py

@@ -40,11 +40,11 @@
 @auth.get(
     "/testsets",
     description="Get Stored TestSets.",
-    response_model=list[schema.TestSets],
+    response_model=list[schema.QASets],
 )
 async def testbed_testsets(
     client: schema.ClientIdType = Header(default="server"),
-) -> list[schema.TestSets]:
+) -> list[schema.QASets]:
     """Get a list of stored TestSets, create TestSet objects if they don't exist"""
     testsets = utils_testbed.get_testsets(db_conn=utils_databases.get_client_database(client).connection)
     return testsets
@@ -56,7 +56,7 @@ async def testbed_testsets(
     response_model=list[schema.Evaluation],
 )
 async def testbed_evaluations(
-    tid: schema.TestSetsIdType,
+    tid: schema.QASetsIdType,
     client: schema.ClientIdType = Header(default="server"),
 ) -> list[schema.Evaluation]:
     """Get Evaluations"""
@@ -72,7 +72,7 @@ async def testbed_evaluations(
     response_model=schema.EvaluationReport,
 )
 async def testbed_evaluation(
-    eid: schema.TestSetsIdType,
+    eid: schema.QASetsIdType,
     client: schema.ClientIdType = Header(default="server"),
 ) -> schema.EvaluationReport:
     """Get Evaluations"""
@@ -84,13 +84,13 @@ async def testbed_evaluation(
 
 @auth.get(
     "/testset_qa",
-    description="Get Stored schema.TestSets Q&A.",
-    response_model=schema.TestSetQA,
+    description="Get Stored Testbed Q&A.",
+    response_model=schema.QASetData,
 )
 async def testbed_testset_qa(
-    tid: schema.TestSetsIdType,
+    tid: schema.QASetsIdType,
     client: schema.ClientIdType = Header(default="server"),
-) -> schema.TestSetQA:
+) -> schema.QASetData:
     """Get TestSet Q&A"""
     return utils_testbed.get_testset_qa(
         db_conn=utils_databases.get_client_database(client).connection, tid=tid.upper()
@@ -102,7 +102,7 @@ async def testbed_testset_qa(
     description="Delete a TestSet",
 )
 async def testbed_delete_testset(
-    tid: Optional[schema.TestSetsIdType] = None,
+    tid: Optional[schema.QASetsIdType] = None,
     client: schema.ClientIdType = Header(default="server"),
 ) -> JSONResponse:
     """Delete TestSet"""
@@ -113,14 +113,14 @@ async def testbed_delete_testset(
 @auth.post(
     "/testset_load",
     description="Upsert TestSets.",
-    response_model=schema.TestSetQA,
+    response_model=schema.QASetData,
 )
 async def testbed_upsert_testsets(
     files: list[UploadFile],
-    name: schema.TestSetsNameType,
-    tid: Optional[schema.TestSetsIdType] = None,
+    name: schema.QASetsNameType,
+    tid: Optional[schema.QASetsIdType] = None,
     client: schema.ClientIdType = Header(default="server"),
-) -> schema.TestSetQA:
+) -> schema.QASetData:
     """Update stored TestSet data"""
     created = datetime.now().isoformat()
     db_conn = utils_databases.get_client_database(client).connection
@@ -194,16 +194,16 @@ def _handle_testset_error(ex: Exception, temp_directory, ll_model: str):
 @auth.post(
     "/testset_generate",
     description="Generate Q&A Test Set.",
-    response_model=schema.TestSetQA,
+    response_model=schema.QASetData,
 )
 async def testbed_generate_qa(
     files: list[UploadFile],
-    name: schema.TestSetsNameType,
+    name: schema.QASetsNameType,
     ll_model: str,
     embed_model: str,
     questions: int = 2,
     client: schema.ClientIdType = Header(default="server"),
-) -> schema.TestSetQA:
+) -> schema.QASetData:
     """Retrieve contents from a local file uploaded and generate Q&A"""
     # Get the Model Configuration
     try:
@@ -249,7 +249,7 @@ async def _collect_testbed_answers(loaded_testset: QATestset, client: str) -> li
     response_model=schema.EvaluationReport,
 )
 async def testbed_evaluate(
-    tid: schema.TestSetsIdType,
+    tid: schema.QASetsIdType,
     judge: str,
     client: schema.ClientIdType = Header(default="server"),
 ) -> schema.EvaluationReport: