diff --git a/docs/howtos/customizations/metrics/_write_your_own_metric.md b/docs/howtos/customizations/metrics/_write_your_own_metric.md
index b5386ceff..4913309b1 100644
--- a/docs/howtos/customizations/metrics/_write_your_own_metric.md
+++ b/docs/howtos/customizations/metrics/_write_your_own_metric.md
@@ -34,7 +34,7 @@ choose_evaluator_llm.md
```python
from ragas.llms import llm_factory
-evaluator_llm = llm_factory('gpt-4o')
+evaluator_llm = llm_factory("gpt-4o")
```
## Aspect Critic - Simple Criteria Scoring
@@ -56,7 +56,7 @@ from ragas.metrics import AspectCritic
hallucinations_binary = AspectCritic(
name="hallucinations_binary",
definition="Did the model hallucinate or add any information that was not present in the retrieved context?",
- llm=evaluator_llm
+ llm=evaluator_llm,
)
await hallucinations_binary.single_turn_ascore(eval_dataset[0])
@@ -93,9 +93,7 @@ Now lets init the metric with the rubric and evaluator llm and evaluate the data
from ragas.metrics import RubricsScoreWithoutReference
hallucinations_rubric = RubricsScoreWithoutReference(
- name="hallucinations_rubric",
- llm=evaluator_llm,
- rubrics=rubric
+ name="hallucinations_rubric", llm=evaluator_llm, rubrics=rubric
)
await hallucinations_rubric.single_turn_ascore(eval_dataset[0])
@@ -125,7 +123,6 @@ For our example, we need to to use LLMs to evaluate our metric so we will subcla
As for the implementation, we will use the [Faithfulness][ragas.metrics.Faithfulness] metric to evaluate our metric to measure the hallucinations with the formula
-
$$
\text{Hallucinations} = 1 - \text{Faithfulness}
$$
@@ -144,19 +141,28 @@ import typing as t
from ragas.callbacks import Callbacks
from ragas.dataset_schema import SingleTurnSample
+
@dataclass
class HallucinationsMetric(MetricWithLLM, SingleTurnMetric):
# name of the metric
name: str = "hallucinations_metric"
# we need to define the required columns for the metric
- _required_columns: t.Dict[MetricType, t.Set[str]] = field(default_factory=lambda: {MetricType.SINGLE_TURN: {"user_input", "response", "retrieved_contexts"}})
+ _required_columns: t.Dict[MetricType, t.Set[str]] = field(
+ default_factory=lambda: {
+ MetricType.SINGLE_TURN: {"user_input", "response", "retrieved_contexts"}
+ }
+ )
def __post_init__(self):
# init the faithfulness metric
self.faithfulness_metric = Faithfulness(llm=self.llm)
- async def _single_turn_ascore(self, sample: SingleTurnSample, callbacks: Callbacks) -> float:
- faithfulness_score = await self.faithfulness_metric.single_turn_ascore(sample, callbacks)
+ async def _single_turn_ascore(
+ self, sample: SingleTurnSample, callbacks: Callbacks
+ ) -> float:
+ faithfulness_score = await self.faithfulness_metric.single_turn_ascore(
+ sample, callbacks
+ )
return 1 - faithfulness_score
```
@@ -181,12 +187,8 @@ Now let's evaluate the entire dataset with the metrics we have created.
from ragas import evaluate
results = evaluate(
- eval_dataset,
- metrics=[
- hallucinations_metric,
- hallucinations_rubric,
- hallucinations_binary
- ],
+ eval_dataset,
+ metrics=[hallucinations_metric, hallucinations_rubric, hallucinations_binary],
)
```
diff --git a/docs/howtos/customizations/testgenerator/_persona_generator.md b/docs/howtos/customizations/testgenerator/_persona_generator.md
new file mode 100644
index 000000000..d4c6d0db0
--- /dev/null
+++ b/docs/howtos/customizations/testgenerator/_persona_generator.md
@@ -0,0 +1,160 @@
+## Persona's in Testset Generation
+
+You can add different persona's to the testset generation process by defining the [Persona][ragas.testset.persona.Persona] class with the name and role description of the different persona's that might be relevant to your use case and you want to generate testset for.
+
+For example, for the [gitlab handbook](https://about.gitlab.com/handbook/) we might want to generate testset for different persona's like a new joinee, a manager, a senior manager, etc. And hence we will define them as follows:
+
+1. New Joinee: Don't know much about the company and is looking for information on how to get started.
+2. Manager: Wants to know about the different teams and how they collaborate with each other.
+3. Senior Manager: Wants to know about the company vision and how it is executed.
+
+Which we can define as follows:
+
+
+```python
+from ragas.testset.persona import Persona
+
+persona_new_joinee = Persona(name="New Joinee", role_description="Don't know much about the company and is looking for information on how to get started.")
+persona_manager = Persona(name="Manager", role_description="Wants to know about the different teams and how they collaborate with each other.")
+persona_senior_manager = Persona(name="Senior Manager", role_description="Wants to know about the company vision and how it is executed.")
+
+personas = [persona_new_joinee, persona_manager, persona_senior_manager]
+personas
+```
+
+
+
+
+ [Persona(name='New Joinee', role_description="Don't know much about the company and is looking for information on how to get started."),
+ Persona(name='Manager', role_description='Wants to know about the different teams and how they collaborate with each other.'),
+ Persona(name='Senior Manager', role_description='Wants to know about the company vision and how it is executed.')]
+
+
+
+And then you can use these persona's in the testset generation process by passing them to the [TestsetGenerator][ragas.testset.generator.TestsetGenerator] class.
+
+
+```python
+from ragas.testset import TestsetGenerator
+from ragas.testset.graph import KnowledgeGraph
+from ragas.llms import llm_factory
+
+# Load the knowledge graph
+kg = KnowledgeGraph.load("../../../../experiments/gitlab_kg.json")
+# Initialize the Generator LLM
+llm = llm_factory("gpt-4o-mini")
+
+# Initialize the Testset Generator
+testset_generator = TestsetGenerator(knowledge_graph=kg, persona_list=personas, llm=llm)
+# Generate the Testset
+testset = testset_generator.generate(testset_size=10)
+testset
+
+```
+
+
+```python
+testset.to_pandas().head()
+```
+
+
+
+
+
+
+
+
+
+ |
+ user_input |
+ reference_contexts |
+ reference |
+ synthesizer_name |
+
+
+
+
+ 0 |
+ What the Director do in GitLab and how they wo... |
+ [09db4f3e-1c10-4863-9024-f869af48d3e0\n\ntitle... |
+ The Director at GitLab, such as the Director o... |
+ single_hop_specifc_query_synthesizer |
+
+
+ 1 |
+ Wht is the rol of the VP in GitLab? |
+ [56c84f1b-3558-4c80-b8a9-348e69a4801b\n\nJob F... |
+ The VP, or Vice President, at GitLab is respon... |
+ single_hop_specifc_query_synthesizer |
+
+
+ 2 |
+ What GitLab do for career progression? |
+ [ead619a5-930f-4e2b-b797-41927a04d2e3\n\nGoals... |
+ The Job frameworks at GitLab help team members... |
+ single_hop_specifc_query_synthesizer |
+
+
+ 3 |
+ Wht is the S-grop and how do they work with ot... |
+ [42babb12-b033-493f-b684-914e2b1b1d0f\n\nPeopl... |
+ Members of the S-group are expected to demonst... |
+ single_hop_specifc_query_synthesizer |
+
+
+ 4 |
+ How does Google execute its company vision? |
+ [c3ed463d-1cdc-4ba4-a6ca-2c4ab12da883\n\nof mo... |
+ To effectively execute the company vision, man... |
+ single_hop_specifc_query_synthesizer |
+
+
+
+
+
+
+
+## Automatic Persona Generation
+
+If you want to automatically generate persona's from a knowledge graph, you can use the [generate_personas_from_kg][ragas.testset.persona.generate_personas_from_kg] function.
+
+
+
+```python
+from ragas.testset.persona import generate_personas_from_kg
+from ragas.testset.graph import KnowledgeGraph
+from ragas.llms import llm_factory
+
+kg = KnowledgeGraph.load("../../../../experiments/gitlab_kg.json")
+llm = llm_factory("gpt-4o-mini")
+
+personas = generate_personas_from_kg(kg=kg, llm=llm, num_personas=5)
+```
+
+
+```python
+personas
+```
+
+
+
+
+ [Persona(name='Organizational Development Manager', role_description='Responsible for implementing job frameworks and career development strategies to enhance employee growth and clarify roles within the company.'),
+ Persona(name='DevSecOps Product Manager', role_description='Responsible for overseeing the development and strategy of DevSecOps solutions, ensuring alignment with company goals and user needs.'),
+ Persona(name='Product Pricing Analyst', role_description='Responsible for developing and analyzing pricing strategies that align with customer needs and market demands.'),
+ Persona(name='Site Reliability Engineer', role_description='Responsible for maintaining service reliability and performance, focusing on implementing rate limits to prevent outages and enhance system stability.'),
+ Persona(name='Security Operations Engineer', role_description="Works on enhancing security logging processes and ensuring compliance within GitLab's infrastructure.")]
+
+
diff --git a/docs/howtos/customizations/testgenerator/persona_generator.ipynb b/docs/howtos/customizations/testgenerator/persona_generator.ipynb
new file mode 100644
index 000000000..7ed8e7744
--- /dev/null
+++ b/docs/howtos/customizations/testgenerator/persona_generator.ipynb
@@ -0,0 +1,241 @@
+{
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Persona's in Testset Generation\n",
+ "\n",
+ "You can add different persona's to the testset generation process by defining the [Persona][ragas.testset.persona.Persona] class with the name and role description of the different persona's that might be relevant to your use case and you want to generate testset for.\n",
+ "\n",
+ "For example, for the [gitlab handbook](https://about.gitlab.com/handbook/) we might want to generate testset for different persona's like a new joinee, a manager, a senior manager, etc. And hence we will define them as follows:\n",
+ "\n",
+ "1. New Joinee: Don't know much about the company and is looking for information on how to get started.\n",
+ "2. Manager: Wants to know about the different teams and how they collaborate with each other.\n",
+ "3. Senior Manager: Wants to know about the company vision and how it is executed.\n",
+ "\n",
+ "Which we can define as follows:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 13,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[Persona(name='New Joinee', role_description=\"Don't know much about the company and is looking for information on how to get started.\"),\n",
+ " Persona(name='Manager', role_description='Wants to know about the different teams and how they collaborate with each other.'),\n",
+ " Persona(name='Senior Manager', role_description='Wants to know about the company vision and how it is executed.')]"
+ ]
+ },
+ "execution_count": 13,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "from ragas.testset.persona import Persona\n",
+ "\n",
+ "persona_new_joinee = Persona(name=\"New Joinee\", role_description=\"Don't know much about the company and is looking for information on how to get started.\")\n",
+ "persona_manager = Persona(name=\"Manager\", role_description=\"Wants to know about the different teams and how they collaborate with each other.\")\n",
+ "persona_senior_manager = Persona(name=\"Senior Manager\", role_description=\"Wants to know about the company vision and how it is executed.\")\n",
+ "\n",
+ "personas = [persona_new_joinee, persona_manager, persona_senior_manager]\n",
+ "personas"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "And then you can use these persona's in the testset generation process by passing them to the [TestsetGenerator][ragas.testset.generator.TestsetGenerator] class."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ragas.testset import TestsetGenerator\n",
+ "from ragas.testset.graph import KnowledgeGraph\n",
+ "from ragas.llms import llm_factory\n",
+ "\n",
+ "# Load the knowledge graph\n",
+ "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n",
+ "# Initialize the Generator LLM\n",
+ "llm = llm_factory(\"gpt-4o-mini\")\n",
+ "\n",
+ "# Initialize the Testset Generator\n",
+ "testset_generator = TestsetGenerator(knowledge_graph=kg, persona_list=personas, llm=llm)\n",
+ "# Generate the Testset\n",
+ "testset = testset_generator.generate(testset_size=10)\n",
+ "testset\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 15,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/html": [
+ "\n",
+ "\n",
+ "
\n",
+ " \n",
+ " \n",
+ " | \n",
+ " user_input | \n",
+ " reference_contexts | \n",
+ " reference | \n",
+ " synthesizer_name | \n",
+ "
\n",
+ " \n",
+ " \n",
+ " \n",
+ " 0 | \n",
+ " What the Director do in GitLab and how they wo... | \n",
+ " [09db4f3e-1c10-4863-9024-f869af48d3e0\\n\\ntitle... | \n",
+ " The Director at GitLab, such as the Director o... | \n",
+ " single_hop_specifc_query_synthesizer | \n",
+ "
\n",
+ " \n",
+ " 1 | \n",
+ " Wht is the rol of the VP in GitLab? | \n",
+ " [56c84f1b-3558-4c80-b8a9-348e69a4801b\\n\\nJob F... | \n",
+ " The VP, or Vice President, at GitLab is respon... | \n",
+ " single_hop_specifc_query_synthesizer | \n",
+ "
\n",
+ " \n",
+ " 2 | \n",
+ " What GitLab do for career progression? | \n",
+ " [ead619a5-930f-4e2b-b797-41927a04d2e3\\n\\nGoals... | \n",
+ " The Job frameworks at GitLab help team members... | \n",
+ " single_hop_specifc_query_synthesizer | \n",
+ "
\n",
+ " \n",
+ " 3 | \n",
+ " Wht is the S-grop and how do they work with ot... | \n",
+ " [42babb12-b033-493f-b684-914e2b1b1d0f\\n\\nPeopl... | \n",
+ " Members of the S-group are expected to demonst... | \n",
+ " single_hop_specifc_query_synthesizer | \n",
+ "
\n",
+ " \n",
+ " 4 | \n",
+ " How does Google execute its company vision? | \n",
+ " [c3ed463d-1cdc-4ba4-a6ca-2c4ab12da883\\n\\nof mo... | \n",
+ " To effectively execute the company vision, man... | \n",
+ " single_hop_specifc_query_synthesizer | \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ " user_input ... synthesizer_name\n",
+ "0 What the Director do in GitLab and how they wo... ... single_hop_specifc_query_synthesizer\n",
+ "1 Wht is the rol of the VP in GitLab? ... single_hop_specifc_query_synthesizer\n",
+ "2 What GitLab do for career progression? ... single_hop_specifc_query_synthesizer\n",
+ "3 Wht is the S-grop and how do they work with ot... ... single_hop_specifc_query_synthesizer\n",
+ "4 How does Google execute its company vision? ... single_hop_specifc_query_synthesizer\n",
+ "\n",
+ "[5 rows x 4 columns]"
+ ]
+ },
+ "execution_count": 15,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "testset.to_pandas().head()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## Automatic Persona Generation\n",
+ "\n",
+ "If you want to automatically generate persona's from a knowledge graph, you can use the [generate_personas_from_kg][ragas.testset.persona.generate_personas_from_kg] function.\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "from ragas.testset.persona import generate_personas_from_kg\n",
+ "from ragas.testset.graph import KnowledgeGraph\n",
+ "from ragas.llms import llm_factory\n",
+ "\n",
+ "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n",
+ "llm = llm_factory(\"gpt-4o-mini\")\n",
+ "\n",
+ "personas = generate_personas_from_kg(kg=kg, llm=llm, num_personas=5)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 12,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "[Persona(name='Organizational Development Manager', role_description='Responsible for implementing job frameworks and career development strategies to enhance employee growth and clarify roles within the company.'),\n",
+ " Persona(name='DevSecOps Product Manager', role_description='Responsible for overseeing the development and strategy of DevSecOps solutions, ensuring alignment with company goals and user needs.'),\n",
+ " Persona(name='Product Pricing Analyst', role_description='Responsible for developing and analyzing pricing strategies that align with customer needs and market demands.'),\n",
+ " Persona(name='Site Reliability Engineer', role_description='Responsible for maintaining service reliability and performance, focusing on implementing rate limits to prevent outages and enhance system stability.'),\n",
+ " Persona(name='Security Operations Engineer', role_description=\"Works on enhancing security logging processes and ensuring compliance within GitLab's infrastructure.\")]"
+ ]
+ },
+ "execution_count": 12,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "personas"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": "ragas",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.10.12"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}
diff --git a/docs/howtos/integrations/_langgraph_agent_evaluation.md b/docs/howtos/integrations/_langgraph_agent_evaluation.md
index 8ffa47a10..800f0678c 100644
--- a/docs/howtos/integrations/_langgraph_agent_evaluation.md
+++ b/docs/howtos/integrations/_langgraph_agent_evaluation.md
@@ -230,7 +230,7 @@ display(Image(react_graph.get_graph(xray=True).draw_mermaid_png()))
-![jpeg](../../_static/imgs/_langgraph_agent_evaluation_28_0.jpg)
+![jpeg](_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg)
diff --git a/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg b/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg
new file mode 100644
index 000000000..e580db4d4
Binary files /dev/null and b/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg differ
diff --git a/docs/howtos/integrations/_llamaindex.md b/docs/howtos/integrations/_llamaindex.md
index abaaa79de..bdff4c78a 100644
--- a/docs/howtos/integrations/_llamaindex.md
+++ b/docs/howtos/integrations/_llamaindex.md
@@ -11,13 +11,15 @@ You will need an testset to evaluate your `QueryEngine` against. You can either
Let's see how that works with Llamaindex
# load the documents
+
+
```python
from llama_index.core import SimpleDirectoryReader
documents = SimpleDirectoryReader("./nyc_wikipedia").load_data()
```
-Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms
+Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms
```python
diff --git a/mkdocs.yml b/mkdocs.yml
index 49954d63d..04a6b81ce 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -84,6 +84,7 @@ nav:
- Write your own Metrics - (advanced): howtos/customizations/metrics/_write_your_own_metric_advanced.md
- Testset Generation:
- howtos/customizations/testgenerator/index.md
+ - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md
- Applications:
- howtos/applications/index.md
- Metrics:
diff --git a/src/ragas/testset/synthesizers/__init__.py b/src/ragas/testset/synthesizers/__init__.py
index 48679a179..7ccde116c 100644
--- a/src/ragas/testset/synthesizers/__init__.py
+++ b/src/ragas/testset/synthesizers/__init__.py
@@ -15,7 +15,6 @@
def default_query_distribution(llm: BaseRagasLLM) -> QueryDistribution:
- """ """
return [
(SingleHopSpecificQuerySynthesizer(llm=llm), 0.5),
(MultiHopAbstractQuerySynthesizer(llm=llm), 0.25),
diff --git a/src/ragas/testset/synthesizers/multi_hop/abstract.py b/src/ragas/testset/synthesizers/multi_hop/abstract.py
index 20162ff4e..65020ddf8 100644
--- a/src/ragas/testset/synthesizers/multi_hop/abstract.py
+++ b/src/ragas/testset/synthesizers/multi_hop/abstract.py
@@ -70,6 +70,10 @@ async def _generate_scenarios(
logger.info("found %d clusters", len(node_clusters))
scenarios = []
+ if len(node_clusters) == 0:
+ raise ValueError(
+ "No clusters found in the knowledge graph. Use a different Synthesizer."
+ )
num_sample_per_cluster = int(np.ceil(n / len(node_clusters)))
for cluster in node_clusters: