(EAI-601): Add retrieval evals from verified answers (#554)

* add distinct retrieveRelevantContent module + braintrust tracing * move retrieval conf * retrieval eval working in correct location * Add avg score * (EAI-383) Fetch & Refine Top Jira Tickets (#539) * 601 start: load VAs as yaml * load from verified answers * manual only evals * revert to all docs * clean up merge conflict * add tests * checkpoint w/ vaToConvo * only VAs in existing retrieval eval cases * remove unused --------- Co-authored-by: Nick Larew <[email protected]>
mongodb · Nov 12, 2024 · 377046f · 377046f
1 parent 6098b7b
commit 377046f
Show file tree

Hide file tree

Showing 2 changed files with 325 additions and 2 deletions.
diff --git a/packages/chatbot-server-mongodb-public/evalCases/included_links_conversations.yml b/packages/chatbot-server-mongodb-public/evalCases/included_links_conversations.yml
@@ -754,3 +754,326 @@
     - reference/operator/aggregation/merge
     - atlas/data-federation/supported-unsupported/pipeline/merge
     - reference/operator/aggregation/mergeObjects
+- name: "Verified Answer: Get started with MongoDB"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: Get started with MongoDB
+  expectedLinks:
+    - https://mongodb.com/docs/manual/tutorial/getting-started/
+    - https://mongodb.com/docs/develop-applications
+    - https://mongodb.com/docs/manual/
+- name: "Verified Answer: How do I install MongoDB on Windows?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: How do I install MongoDB on Windows?
+  expectedLinks:
+    - https://mongodb.com/docs/manual/tutorial/install-mongodb-on-windows-unattended/
+    - https://mongodb.com/docs/manual/tutorial/install-mongodb-on-windows/
+    - https://mongodb.com/docs/manual/tutorial/install-mongodb-enterprise-on-windows/
+    - https://mongodb.com/docs/manual/installation/
+- name: "Verified Answer: How do I insert data into MongoDB?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: How do I insert data into MongoDB?
+  expectedLinks:
+    - https://mongodb.com/docs/guides/crud/insert/
+    - https://mongodb.com/docs/atlas/tutorial/insert-data-into-your-cluster/
+- name: "Verified Answer: command to create new collection"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: command to create new collection
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/method/db.createCollection/
+    - https://mongodb.com/docs/manual/core/databases-and-collections/
+- name: "Verified Answer: What is mongodb?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: What is mongodb?
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/introduction/
+    - https://mongodb.com/docs/manual/
+- name: "Verified Answer: updateMany"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: updateMany
+  expectedLinks:
+    - https://mongodb.com/docs/manual/tutorial/update-documents/
+    - https://mongodb.com/docs/drivers/node/current/usage-examples/updateMany/
+- name: "Verified Answer: countDocuments"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: countDocuments
+  expectedLinks:
+    - https://mongodb.com/docs/drivers/node/current/usage-examples/count/
+    - https://mongodb.com/docs/manual/reference/method/db.collection.countDocuments/
+- name: "Verified Answer: how to query"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to query
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/tutorial/query-documents/
+    - https://www.mongodb.com/docs/manual/reference/method/db.collection.find/
+    - https://mongodb.com/docs/guides/crud/read_queries/
+- name: "Verified Answer: how to delete data"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to delete data
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/reference/command/delete
+    - https://www.mongodb.com/docs/manual/tutorial/remove-documents/
+    - https://mongodb.com/docs/guides/crud/delete/
+- name: "Verified Answer: delete one document"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: delete one document
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/tutorial/remove-documents/
+    - https://mongodb.com/docs/guides/crud/delete/
+    - https://mongodb.com/docs/drivers/node/current/usage-examples/deleteOne/
+- name: "Verified Answer: How do I backup a MongoDB database?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: How do I backup a MongoDB database?
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/core/backups/
+    - https://www.mongodb.com/docs/atlas/backup-restore-cluster/
+    - https://www.mongodb.com/docs/manual/tutorial/backup-and-restore-tools/
+- name: "Verified Answer: What is aggregation in MongoDB?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: What is aggregation in MongoDB?
+  expectedLinks:
+    - https://www.mongodb.com/docs/atlas/atlas-ui/agg-pipeline/
+    - https://www.mongodb.com/docs/manual/core/aggregation-pipeline/
+    - https://www.mongodb.com/developer/products/mongodb/introduction-aggregation-framework
+- name: "Verified Answer: aggregation framework"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: aggregation framework
+  expectedLinks:
+    - https://www.mongodb.com/docs/atlas/atlas-ui/agg-pipeline/
+    - https://www.mongodb.com/docs/manual/core/aggregation-pipeline/
+    - https://www.mongodb.com/developer/products/mongodb/introduction-aggregation-framework
+- name: "Verified Answer: mongoshell"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: mongoshell
+  expectedLinks:
+    - https://mongodb.com/docs/mongodb-shell/
+- name: "Verified Answer: how to remove object in array in mongoDB"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to remove object in array in mongoDB
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/operator/update/pop/
+    - https://mongodb.com/docs/manual/reference/operator/update/pull/
+- name: "Verified Answer: distinct"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: distinct
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/command/distinct/
+    - https://mongodb.com/docs/manual/reference/method/db.collection.distinct/
+- name: "Verified Answer: how to find unique values in my fields"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to find unique values in my fields
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/reference/command/distinct/
+    - https://mongodb.com/docs/manual/reference/method/db.collection.distinct/
+- name: "Verified Answer: how to change maxBsonObjectSize"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to change maxBsonObjectSize
+  expectedLinks:
+    - https://www.mongodb.com/docs/manual/reference/limits
+- name: "Verified Answer: how to update document"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to update document
+  expectedLinks:
+    - https://mongodb.com/docs/drivers/node/current/usage-examples/updateOne/
+    - https://mongodb.com/docs/manual/tutorial/update-documents/
+    - https://mongodb.com/docs/drivers/node/current/fundamentals/crud/write-operations/modify/
+- name: "Verified Answer: how to import data"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to import data
+  expectedLinks:
+    - https://www.mongodb.com/developer/products/mongodb/mongoimport-guide
+    - https://mongodb.com/docs/guides/crud/insert/
+- name: "Verified Answer: operators in mongodb"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: operators in mongodb
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/operator/aggregation/
+    - https://mongodb.com/docs/manual/reference/operator/query/
+    - https://mongodb.com/docs/manual/release-notes/6.0/
+    - https://mongodb.com/docs/manual/reference/operator/update/
+- name: "Verified Answer: how do I find with a projection"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how do I find with a projection
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/method/db.collection.find/
+    - https://mongodb.com/docs/manual/reference/method/db.collection.findOne/
+- name: "Verified Answer: how can i find a document by id"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how can i find a document by id
+  expectedLinks:
+    - https://mongodb.com/docs/drivers/node/current/usage-examples/findOne/
+- name: "Verified Answer: connection string"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: connection string
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/connection-string/
+- name: "Verified Answer: how many authentication methods for MongoDB?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how many authentication methods for MongoDB?
+  expectedLinks:
+    - https://mongodb.com/docs/manual/core/authentication/
+    - https://mongodb.com/docs/drivers/node/current/fundamentals/authentication/mechanisms/
+    - https://mongodb.com/docs/drivers/node/current/fundamentals/authentication/
+- name: "Verified Answer: how to setup replica cluster"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: how to setup replica cluster
+  expectedLinks:
+    - https://mongodb.com/docs/manual/tutorial/deploy-replica-set/
+- name: "Verified Answer: give me an example of how to use the $and operator"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: give me an example of how to use the $and operator
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/operator/query/and/
+    - https://mongodb.com/docs/manual/reference/operator/aggregation/and/
+
+- name: "Verified Answer: give me an example of how to use the $in operator"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: give me an example of how to use the $in operator
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/operator/query/in/
+    - https://mongodb.com/docs/manual/reference/operator/aggregation/in/
+    - https://mongodb.com/docs/atlas/atlas-search/in/
+- name: "Verified Answer: Can you please give me a mongodb aggregation pipeline
+    which does a $lookup and $match"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content:
+        Can you please give me a mongodb aggregation pipeline which does a
+        $lookup and $match
+  expectedLinks:
+    - https://mongodb.com/docs/manual/reference/operator/aggregation/lookup/
+    - https://mongodb.com/docs/manual/reference/operator/aggregation/graphLookup/
+    - https://www.mongodb.com/docs/manual/reference/operator/aggregation/match/
+- name: "Verified Answer: Can i reset my password"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: Can i reset my password
+  expectedLinks:
+    - https://mongodb.com/docs/atlas/security/manage-your-mongodb-atlas-account/
+    - https://mongodb.com/docs/manual/tutorial/change-own-password-and-custom-data/
+- name: "Verified Answer: Password reset"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: Password reset
+  expectedLinks:
+    - https://mongodb.com/docs/atlas/security/manage-your-mongodb-atlas-account/
+    - https://mongodb.com/docs/manual/tutorial/change-own-password-and-custom-data/
+- name: "Verified Answer: How do I import or migrate data into MongoDB?"
+  tags:
+    - verified_answer
+  messages:
+    - role: user
+      content: How do I import or migrate data into MongoDB?
+  expectedLinks:
+    - https://www.mongodb.com/developer/products/mongodb/mongoimport-guide
+    - https://learn.mongodb.com/learn/course/importing-csv-data-into-mongodb/learning-byte/learn
+- name: "Verified Answer: What should I learn first on MongoDB University?"
+  tags:
+    - verified_answer
+    - university
+  messages:
+    - role: user
+      content: What should I learn first on MongoDB University?
+  expectedLinks:
+    - https://learn.mongodb.com/learning-paths/introduction-to-mongodb
+    - https://learn.mongodb.com/pages/mongodb-associate-developer-exam
+- name: "Verified Answer: How do I learn about AI and vector search on MongoDB
+    University?"
+  tags:
+    - verified_answer
+    - university
+  messages:
+    - role: user
+      content: How do I learn about AI and vector search on MongoDB University?
+  expectedLinks:
+    - https://learn.mongodb.com/learning-paths/building-genai-apps-learning-badge-path
+    - https://learn.mongodb.com/learn/learning-path/atlas-vector-search
diff --git a/packages/chatbot-server-mongodb-public/src/processors/retrieveRelevantContent.eval.ts b/packages/chatbot-server-mongodb-public/src/processors/retrieveRelevantContent.eval.ts
@@ -62,7 +62,7 @@ type RetrievalEvalScorer = EvalScorer<
 // we could readdress this.
 const { k } = retrievalConfig.findNearestNeighborsOptions;
 
-const simpleConversationEvalTask: EvalTask<
+const retrieveRelevantContentEvalTask: EvalTask<
   RetrievalEvalCaseInput,
   RetrievalTaskOutput
 > = async function (data) {
@@ -203,7 +203,7 @@ Eval("mongodb-chatbot-retrieval", {
   },
   maxConcurrency: 5,
   data: getConversationRetrievalEvalData,
-  task: simpleConversationEvalTask,
+  task: retrieveRelevantContentEvalTask,
   scores: [
     BinaryNdcgAtK,
     F1AtK,