diff --git a/docs/howtos/customizations/metrics/_write_your_own_metric.md b/docs/howtos/customizations/metrics/_write_your_own_metric.md index b5386ceff..4913309b1 100644 --- a/docs/howtos/customizations/metrics/_write_your_own_metric.md +++ b/docs/howtos/customizations/metrics/_write_your_own_metric.md @@ -34,7 +34,7 @@ choose_evaluator_llm.md ```python from ragas.llms import llm_factory -evaluator_llm = llm_factory('gpt-4o') +evaluator_llm = llm_factory("gpt-4o") ``` ## Aspect Critic - Simple Criteria Scoring @@ -56,7 +56,7 @@ from ragas.metrics import AspectCritic hallucinations_binary = AspectCritic( name="hallucinations_binary", definition="Did the model hallucinate or add any information that was not present in the retrieved context?", - llm=evaluator_llm + llm=evaluator_llm, ) await hallucinations_binary.single_turn_ascore(eval_dataset[0]) @@ -93,9 +93,7 @@ Now lets init the metric with the rubric and evaluator llm and evaluate the data from ragas.metrics import RubricsScoreWithoutReference hallucinations_rubric = RubricsScoreWithoutReference( - name="hallucinations_rubric", - llm=evaluator_llm, - rubrics=rubric + name="hallucinations_rubric", llm=evaluator_llm, rubrics=rubric ) await hallucinations_rubric.single_turn_ascore(eval_dataset[0]) @@ -125,7 +123,6 @@ For our example, we need to to use LLMs to evaluate our metric so we will subcla As for the implementation, we will use the [Faithfulness][ragas.metrics.Faithfulness] metric to evaluate our metric to measure the hallucinations with the formula - $$ \text{Hallucinations} = 1 - \text{Faithfulness} $$ @@ -144,19 +141,28 @@ import typing as t from ragas.callbacks import Callbacks from ragas.dataset_schema import SingleTurnSample + @dataclass class HallucinationsMetric(MetricWithLLM, SingleTurnMetric): # name of the metric name: str = "hallucinations_metric" # we need to define the required columns for the metric - _required_columns: t.Dict[MetricType, t.Set[str]] = field(default_factory=lambda: {MetricType.SINGLE_TURN: {"user_input", "response", "retrieved_contexts"}}) + _required_columns: t.Dict[MetricType, t.Set[str]] = field( + default_factory=lambda: { + MetricType.SINGLE_TURN: {"user_input", "response", "retrieved_contexts"} + } + ) def __post_init__(self): # init the faithfulness metric self.faithfulness_metric = Faithfulness(llm=self.llm) - async def _single_turn_ascore(self, sample: SingleTurnSample, callbacks: Callbacks) -> float: - faithfulness_score = await self.faithfulness_metric.single_turn_ascore(sample, callbacks) + async def _single_turn_ascore( + self, sample: SingleTurnSample, callbacks: Callbacks + ) -> float: + faithfulness_score = await self.faithfulness_metric.single_turn_ascore( + sample, callbacks + ) return 1 - faithfulness_score ``` @@ -181,12 +187,8 @@ Now let's evaluate the entire dataset with the metrics we have created. from ragas import evaluate results = evaluate( - eval_dataset, - metrics=[ - hallucinations_metric, - hallucinations_rubric, - hallucinations_binary - ], + eval_dataset, + metrics=[hallucinations_metric, hallucinations_rubric, hallucinations_binary], ) ``` diff --git a/docs/howtos/customizations/testgenerator/_persona_generator.md b/docs/howtos/customizations/testgenerator/_persona_generator.md new file mode 100644 index 000000000..d4c6d0db0 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/_persona_generator.md @@ -0,0 +1,160 @@ +## Persona's in Testset Generation + +You can add different persona's to the testset generation process by defining the [Persona][ragas.testset.persona.Persona] class with the name and role description of the different persona's that might be relevant to your use case and you want to generate testset for. + +For example, for the [gitlab handbook](https://about.gitlab.com/handbook/) we might want to generate testset for different persona's like a new joinee, a manager, a senior manager, etc. And hence we will define them as follows: + +1. New Joinee: Don't know much about the company and is looking for information on how to get started. +2. Manager: Wants to know about the different teams and how they collaborate with each other. +3. Senior Manager: Wants to know about the company vision and how it is executed. + +Which we can define as follows: + + +```python +from ragas.testset.persona import Persona + +persona_new_joinee = Persona(name="New Joinee", role_description="Don't know much about the company and is looking for information on how to get started.") +persona_manager = Persona(name="Manager", role_description="Wants to know about the different teams and how they collaborate with each other.") +persona_senior_manager = Persona(name="Senior Manager", role_description="Wants to know about the company vision and how it is executed.") + +personas = [persona_new_joinee, persona_manager, persona_senior_manager] +personas +``` + + + + + [Persona(name='New Joinee', role_description="Don't know much about the company and is looking for information on how to get started."), + Persona(name='Manager', role_description='Wants to know about the different teams and how they collaborate with each other.'), + Persona(name='Senior Manager', role_description='Wants to know about the company vision and how it is executed.')] + + + +And then you can use these persona's in the testset generation process by passing them to the [TestsetGenerator][ragas.testset.generator.TestsetGenerator] class. + + +```python +from ragas.testset import TestsetGenerator +from ragas.testset.graph import KnowledgeGraph +from ragas.llms import llm_factory + +# Load the knowledge graph +kg = KnowledgeGraph.load("../../../../experiments/gitlab_kg.json") +# Initialize the Generator LLM +llm = llm_factory("gpt-4o-mini") + +# Initialize the Testset Generator +testset_generator = TestsetGenerator(knowledge_graph=kg, persona_list=personas, llm=llm) +# Generate the Testset +testset = testset_generator.generate(testset_size=10) +testset + +``` + + +```python +testset.to_pandas().head() +``` + + + + +
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
user_inputreference_contextsreferencesynthesizer_name
0What the Director do in GitLab and how they wo...[09db4f3e-1c10-4863-9024-f869af48d3e0\n\ntitle...The Director at GitLab, such as the Director o...single_hop_specifc_query_synthesizer
1Wht is the rol of the VP in GitLab?[56c84f1b-3558-4c80-b8a9-348e69a4801b\n\nJob F...The VP, or Vice President, at GitLab is respon...single_hop_specifc_query_synthesizer
2What GitLab do for career progression?[ead619a5-930f-4e2b-b797-41927a04d2e3\n\nGoals...The Job frameworks at GitLab help team members...single_hop_specifc_query_synthesizer
3Wht is the S-grop and how do they work with ot...[42babb12-b033-493f-b684-914e2b1b1d0f\n\nPeopl...Members of the S-group are expected to demonst...single_hop_specifc_query_synthesizer
4How does Google execute its company vision?[c3ed463d-1cdc-4ba4-a6ca-2c4ab12da883\n\nof mo...To effectively execute the company vision, man...single_hop_specifc_query_synthesizer
+
+ + + +## Automatic Persona Generation + +If you want to automatically generate persona's from a knowledge graph, you can use the [generate_personas_from_kg][ragas.testset.persona.generate_personas_from_kg] function. + + + +```python +from ragas.testset.persona import generate_personas_from_kg +from ragas.testset.graph import KnowledgeGraph +from ragas.llms import llm_factory + +kg = KnowledgeGraph.load("../../../../experiments/gitlab_kg.json") +llm = llm_factory("gpt-4o-mini") + +personas = generate_personas_from_kg(kg=kg, llm=llm, num_personas=5) +``` + + +```python +personas +``` + + + + + [Persona(name='Organizational Development Manager', role_description='Responsible for implementing job frameworks and career development strategies to enhance employee growth and clarify roles within the company.'), + Persona(name='DevSecOps Product Manager', role_description='Responsible for overseeing the development and strategy of DevSecOps solutions, ensuring alignment with company goals and user needs.'), + Persona(name='Product Pricing Analyst', role_description='Responsible for developing and analyzing pricing strategies that align with customer needs and market demands.'), + Persona(name='Site Reliability Engineer', role_description='Responsible for maintaining service reliability and performance, focusing on implementing rate limits to prevent outages and enhance system stability.'), + Persona(name='Security Operations Engineer', role_description="Works on enhancing security logging processes and ensuring compliance within GitLab's infrastructure.")] + + diff --git a/docs/howtos/customizations/testgenerator/persona_generator.ipynb b/docs/howtos/customizations/testgenerator/persona_generator.ipynb new file mode 100644 index 000000000..7ed8e7744 --- /dev/null +++ b/docs/howtos/customizations/testgenerator/persona_generator.ipynb @@ -0,0 +1,241 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Persona's in Testset Generation\n", + "\n", + "You can add different persona's to the testset generation process by defining the [Persona][ragas.testset.persona.Persona] class with the name and role description of the different persona's that might be relevant to your use case and you want to generate testset for.\n", + "\n", + "For example, for the [gitlab handbook](https://about.gitlab.com/handbook/) we might want to generate testset for different persona's like a new joinee, a manager, a senior manager, etc. And hence we will define them as follows:\n", + "\n", + "1. New Joinee: Don't know much about the company and is looking for information on how to get started.\n", + "2. Manager: Wants to know about the different teams and how they collaborate with each other.\n", + "3. Senior Manager: Wants to know about the company vision and how it is executed.\n", + "\n", + "Which we can define as follows:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Persona(name='New Joinee', role_description=\"Don't know much about the company and is looking for information on how to get started.\"),\n", + " Persona(name='Manager', role_description='Wants to know about the different teams and how they collaborate with each other.'),\n", + " Persona(name='Senior Manager', role_description='Wants to know about the company vision and how it is executed.')]" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "from ragas.testset.persona import Persona\n", + "\n", + "persona_new_joinee = Persona(name=\"New Joinee\", role_description=\"Don't know much about the company and is looking for information on how to get started.\")\n", + "persona_manager = Persona(name=\"Manager\", role_description=\"Wants to know about the different teams and how they collaborate with each other.\")\n", + "persona_senior_manager = Persona(name=\"Senior Manager\", role_description=\"Wants to know about the company vision and how it is executed.\")\n", + "\n", + "personas = [persona_new_joinee, persona_manager, persona_senior_manager]\n", + "personas" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And then you can use these persona's in the testset generation process by passing them to the [TestsetGenerator][ragas.testset.generator.TestsetGenerator] class." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset import TestsetGenerator\n", + "from ragas.testset.graph import KnowledgeGraph\n", + "from ragas.llms import llm_factory\n", + "\n", + "# Load the knowledge graph\n", + "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n", + "# Initialize the Generator LLM\n", + "llm = llm_factory(\"gpt-4o-mini\")\n", + "\n", + "# Initialize the Testset Generator\n", + "testset_generator = TestsetGenerator(knowledge_graph=kg, persona_list=personas, llm=llm)\n", + "# Generate the Testset\n", + "testset = testset_generator.generate(testset_size=10)\n", + "testset\n" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
user_inputreference_contextsreferencesynthesizer_name
0What the Director do in GitLab and how they wo...[09db4f3e-1c10-4863-9024-f869af48d3e0\\n\\ntitle...The Director at GitLab, such as the Director o...single_hop_specifc_query_synthesizer
1Wht is the rol of the VP in GitLab?[56c84f1b-3558-4c80-b8a9-348e69a4801b\\n\\nJob F...The VP, or Vice President, at GitLab is respon...single_hop_specifc_query_synthesizer
2What GitLab do for career progression?[ead619a5-930f-4e2b-b797-41927a04d2e3\\n\\nGoals...The Job frameworks at GitLab help team members...single_hop_specifc_query_synthesizer
3Wht is the S-grop and how do they work with ot...[42babb12-b033-493f-b684-914e2b1b1d0f\\n\\nPeopl...Members of the S-group are expected to demonst...single_hop_specifc_query_synthesizer
4How does Google execute its company vision?[c3ed463d-1cdc-4ba4-a6ca-2c4ab12da883\\n\\nof mo...To effectively execute the company vision, man...single_hop_specifc_query_synthesizer
\n", + "
" + ], + "text/plain": [ + " user_input ... synthesizer_name\n", + "0 What the Director do in GitLab and how they wo... ... single_hop_specifc_query_synthesizer\n", + "1 Wht is the rol of the VP in GitLab? ... single_hop_specifc_query_synthesizer\n", + "2 What GitLab do for career progression? ... single_hop_specifc_query_synthesizer\n", + "3 Wht is the S-grop and how do they work with ot... ... single_hop_specifc_query_synthesizer\n", + "4 How does Google execute its company vision? ... single_hop_specifc_query_synthesizer\n", + "\n", + "[5 rows x 4 columns]" + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "testset.to_pandas().head()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Automatic Persona Generation\n", + "\n", + "If you want to automatically generate persona's from a knowledge graph, you can use the [generate_personas_from_kg][ragas.testset.persona.generate_personas_from_kg] function.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.testset.persona import generate_personas_from_kg\n", + "from ragas.testset.graph import KnowledgeGraph\n", + "from ragas.llms import llm_factory\n", + "\n", + "kg = KnowledgeGraph.load(\"../../../../experiments/gitlab_kg.json\")\n", + "llm = llm_factory(\"gpt-4o-mini\")\n", + "\n", + "personas = generate_personas_from_kg(kg=kg, llm=llm, num_personas=5)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "[Persona(name='Organizational Development Manager', role_description='Responsible for implementing job frameworks and career development strategies to enhance employee growth and clarify roles within the company.'),\n", + " Persona(name='DevSecOps Product Manager', role_description='Responsible for overseeing the development and strategy of DevSecOps solutions, ensuring alignment with company goals and user needs.'),\n", + " Persona(name='Product Pricing Analyst', role_description='Responsible for developing and analyzing pricing strategies that align with customer needs and market demands.'),\n", + " Persona(name='Site Reliability Engineer', role_description='Responsible for maintaining service reliability and performance, focusing on implementing rate limits to prevent outages and enhance system stability.'),\n", + " Persona(name='Security Operations Engineer', role_description=\"Works on enhancing security logging processes and ensuring compliance within GitLab's infrastructure.\")]" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "personas" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/docs/howtos/integrations/_langgraph_agent_evaluation.md b/docs/howtos/integrations/_langgraph_agent_evaluation.md index 8ffa47a10..800f0678c 100644 --- a/docs/howtos/integrations/_langgraph_agent_evaluation.md +++ b/docs/howtos/integrations/_langgraph_agent_evaluation.md @@ -230,7 +230,7 @@ display(Image(react_graph.get_graph(xray=True).draw_mermaid_png())) -![jpeg](../../_static/imgs/_langgraph_agent_evaluation_28_0.jpg) +![jpeg](_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg) diff --git a/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg b/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg new file mode 100644 index 000000000..e580db4d4 Binary files /dev/null and b/docs/howtos/integrations/_langgraph_agent_evaluation_files/_langgraph_agent_evaluation_23_0.jpg differ diff --git a/docs/howtos/integrations/_llamaindex.md b/docs/howtos/integrations/_llamaindex.md index abaaa79de..bdff4c78a 100644 --- a/docs/howtos/integrations/_llamaindex.md +++ b/docs/howtos/integrations/_llamaindex.md @@ -11,13 +11,15 @@ You will need an testset to evaluate your `QueryEngine` against. You can either Let's see how that works with Llamaindex # load the documents + + ```python from llama_index.core import SimpleDirectoryReader documents = SimpleDirectoryReader("./nyc_wikipedia").load_data() ``` -Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms +Now lets init the `TestsetGenerator` object with the corresponding generator and critic llms ```python diff --git a/mkdocs.yml b/mkdocs.yml index 49954d63d..04a6b81ce 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -84,6 +84,7 @@ nav: - Write your own Metrics - (advanced): howtos/customizations/metrics/_write_your_own_metric_advanced.md - Testset Generation: - howtos/customizations/testgenerator/index.md + - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md - Applications: - howtos/applications/index.md - Metrics: diff --git a/src/ragas/testset/synthesizers/__init__.py b/src/ragas/testset/synthesizers/__init__.py index 48679a179..7ccde116c 100644 --- a/src/ragas/testset/synthesizers/__init__.py +++ b/src/ragas/testset/synthesizers/__init__.py @@ -15,7 +15,6 @@ def default_query_distribution(llm: BaseRagasLLM) -> QueryDistribution: - """ """ return [ (SingleHopSpecificQuerySynthesizer(llm=llm), 0.5), (MultiHopAbstractQuerySynthesizer(llm=llm), 0.25), diff --git a/src/ragas/testset/synthesizers/multi_hop/abstract.py b/src/ragas/testset/synthesizers/multi_hop/abstract.py index 20162ff4e..65020ddf8 100644 --- a/src/ragas/testset/synthesizers/multi_hop/abstract.py +++ b/src/ragas/testset/synthesizers/multi_hop/abstract.py @@ -70,6 +70,10 @@ async def _generate_scenarios( logger.info("found %d clusters", len(node_clusters)) scenarios = [] + if len(node_clusters) == 0: + raise ValueError( + "No clusters found in the knowledge graph. Use a different Synthesizer." + ) num_sample_per_cluster = int(np.ceil(n / len(node_clusters))) for cluster in node_clusters: