diff --git a/docs/_static/edited_chain_runs.json b/docs/_static/edited_chain_runs.json new file mode 100644 index 000000000..78c5925fa --- /dev/null +++ b/docs/_static/edited_chain_runs.json @@ -0,0 +1,490 @@ +{ + "answer_correctness": [ + { + "metric_input": { + "user_input": "What is the Theory of Sensory Adaptation and how does it affect human perception?", + "response": "The Theory of Sensory Adaptation is like when your nose stops noticing a strong smell after a while. It means our senses get used to things that stay the same, so we can pay more attention to new things around us. This helps us save energy and notice important changes, like if something new happens!", + "reference": "The Theory of Sensory Adaptation refers to the process by which our sensory receptors become less sensitive to constant stimuli over time. This phenomenon allows individuals to focus on changes in their environment rather than constant, unchanging stimuli. For example, when you enter a room with a strong odor, you may initially notice the smell intensely, but after a while, you become less aware of it. This adaptation is crucial for survival as it helps conserve energy and allows the brain to detect new and potentially important changes in the environment, enhancing our ability to respond to new stimuli." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Sensory Adaptation and how does it affect human perception?", + "response": "The Theory of Sensory Adaptation is like when your nose stops noticing a strong smell after a while. It means our senses get used to things that stay the same, so we can pay more attention to new things around us. This helps us save energy and notice important changes, like if something new happens!", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Sensory Adaptation refers to the process by which our sensory receptors become less sensitive to constant stimuli over time. This phenomenon allows individuals to focus on changes in their environment rather than constant, unchanging stimuli. For example, when you enter a room with a strong odor, you may initially notice the smell intensely, but after a while, you become less aware of it. This adaptation is crucial for survival as it helps conserve energy and allows the brain to detect new and potentially important changes in the environment, enhancing our ability to respond to new stimuli." + }, + "prompt_output": { + "reason": "The response accurately explains the Theory of Sensory Adaptation and its effects on human perception, aligning well with the reference provided.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Evolutionary Fitness and how does it explain the survival of species?", + "response": "The Theory of Evolutionary Fitness is like a game where animals and plants try to be the best at living in their homes. If they have special tricks or traits that help them survive and have babies, they win by passing on their tricks to their kids. It's not just about being strong or fast; it's also about being smart, having babies, and finding food. Over time, nature picks the best tricks, and that's how animals and plants change and get better at living.", + "reference": "The Theory of Evolutionary Fitness is a concept within evolutionary biology that explains how certain traits or behaviors increase an organism's chances of survival and reproduction in a given environment. It is based on the idea that individuals with higher fitness are more likely to pass on their genes to the next generation. Fitness is not just about physical strength or speed; it encompasses a range of factors including adaptability, reproductive success, and the ability to find resources. Over time, natural selection favors traits that enhance fitness, leading to the evolution of species." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Evolutionary Fitness and how does it explain the survival of species?", + "response": "The Theory of Evolutionary Fitness is like a game where animals and plants try to be the best at living in their homes. If they have special tricks or traits that help them survive and have babies, they win by passing on their tricks to their kids. It's not just about being strong or fast; it's also about being smart, having babies, and finding food. Over time, nature picks the best tricks, and that's how animals and plants change and get better at living.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Evolutionary Fitness is a concept within evolutionary biology that explains how certain traits or behaviors increase an organism's chances of survival and reproduction in a given environment. It is based on the idea that individuals with higher fitness are more likely to pass on their genes to the next generation. Fitness is not just about physical strength or speed; it encompasses a range of factors including adaptability, reproductive success, and the ability to find resources. Over time, natural selection favors traits that enhance fitness, leading to the evolution of species." + }, + "prompt_output": { + "reason": "The response accurately explains the Theory of Evolutionary Fitness, covering key aspects such as survival, reproduction, and the role of natural selection, which aligns well with the reference.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Sediment Transport and how does it apply to river engineering?", + "response": "The Theory of Sediment Transport is like a big idea that helps us understand how tiny bits like sand and gravel move around in air. This is super important for people who work with rivers because it helps them figure out how rivers change shape over time. By knowing how fast the water is and how big the bits are, they can make sure rivers don't wash away too much land and stay easy to travel on.", + "reference": "The Theory of Sediment Transport is a fundamental concept in geomorphology and hydraulic engineering that describes the movement of sediment particles, such as sand, gravel, and silt, by fluid flow, typically water. This theory is crucial in river engineering as it helps in understanding and predicting how rivers shape their channels and floodplains over time. It involves the study of sediment erosion, transport, and deposition processes, which are influenced by factors like water velocity, sediment size, and riverbed composition. Engineers use this theory to design sustainable river management practices, prevent erosion, and maintain navigable waterways." + }, + "metric_output": 0, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Sediment Transport and how does it apply to river engineering?", + "response": "The Theory of Sediment Transport is like a big idea that helps us understand how tiny bits like sand and gravel move around in air. This is super important for people who work with rivers because it helps them figure out how rivers change shape over time. By knowing how fast the water is and how big the bits are, they can make sure rivers don't wash away too much land and stay easy to travel on.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Sediment Transport is a fundamental concept in geomorphology and hydraulic engineering that describes the movement of sediment particles, such as sand, gravel, and silt, by fluid flow, typically water. This theory is crucial in river engineering as it helps in understanding and predicting how rivers shape their channels and floodplains over time. It involves the study of sediment erosion, transport, and deposition processes, which are influenced by factors like water velocity, sediment size, and riverbed composition. Engineers use this theory to design sustainable river management practices, prevent erosion, and maintain navigable waterways." + }, + "prompt_output": { + "reason": "The response does not accurately capture the technical aspects of the Theory of Sediment Transport as described in the reference. It oversimplifies the concept and lacks details about sediment erosion, transport, deposition processes, and their implications for river engineering.", + "verdict": 0 + }, + "is_accepted": true, + "edited_output": { + "reason": "Instead of movement through water, the response mentions movement through the air which is wrong.", + "verdict": 0 + } + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Isostasy and how does it explain the Earth's crust equilibrium?", + "response": "The Theory of Isostasy is like saying the Earth's crust is floating on a squishy layer underneath, just like how an iceberg floats on water. This helps explain why some parts of the Earth, like mountains, are higher, and others, like ocean floors, are lower. If a part of the crust is heavy, it sinks a bit, and if it's light, it floats up more. This balancing act happens very quickly over a short time and helps make things like mountains and changes after big ice sheets melt.", + "reference": "The Theory of Isostasy is a concept in geology that explains the equilibrium of the Earth's crust. It suggests that the Earth's lithosphere floats on the more fluid asthenosphere beneath it, much like an iceberg floats on water. This theory accounts for the varying elevations of the Earth's surface, such as mountains and ocean basins. According to isostasy, areas of the crust that are thicker or denser will sink deeper into the mantle, while lighter or thinner areas will rise. This balance is achieved over geological time scales and is responsible for the vertical movements of the Earth's crust, contributing to phenomena such as post-glacial rebound and the formation of mountain ranges." + }, + "metric_output": 0, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Isostasy and how does it explain the Earth's crust equilibrium?", + "response": "The Theory of Isostasy is like saying the Earth's crust is floating on a squishy layer underneath, just like how an iceberg floats on water. This helps explain why some parts of the Earth, like mountains, are higher, and others, like ocean floors, are lower. If a part of the crust is heavy, it sinks a bit, and if it's light, it floats up more. This balancing act happens very quickly over a short time and helps make things like mountains and changes after big ice sheets melt.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Isostasy is a concept in geology that explains the equilibrium of the Earth's crust. It suggests that the Earth's lithosphere floats on the more fluid asthenosphere beneath it, much like an iceberg floats on water. This theory accounts for the varying elevations of the Earth's surface, such as mountains and ocean basins. According to isostasy, areas of the crust that are thicker or denser will sink deeper into the mantle, while lighter or thinner areas will rise. This balance is achieved over geological time scales and is responsible for the vertical movements of the Earth's crust, contributing to phenomena such as post-glacial rebound and the formation of mountain ranges." + }, + "prompt_output": { + "reason": "The response provides a simplified explanation of the Theory of Isostasy but lacks the depth and detail present in the reference. It incorrectly states that the balancing act happens very quickly over a short time, while the reference emphasizes that this balance is achieved over geological time scales.", + "verdict": 0 + }, + "is_accepted": true, + "edited_output": { + "reason": "It incorrectly states that the balancing act happens very quickly over a short time, while the reference emphasizes that this balance is achieved over geological time scales.", + "verdict": 0 + } + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What are the key concepts in the Theory of Digital Computation?", + "response": "The Theory of Digital Computation is like a big toolbox for understanding computers. It has things like algorithms, which are like recipes for solving problems. It also looks at how hard these problems are to solve, called computational complexity. There are also ideas about pretend machines, called automata, and how they work. Formal languages are like special codes that computers understand. Turing machines are like imaginary computers that help us think about what computers can do. The Church-Turing thesis says that these pretend computers can do anything a real computer can do. All these ideas help us know what computers can and can't do.", + "reference": "The Theory of Digital Computation encompasses several key concepts that form the foundation of computer science. These include algorithms, which are step-by-step procedures for solving problems; computational complexity, which studies the resources required for algorithms to solve problems; automata theory, which examines abstract machines and the problems they can solve; and formal languages, which are sets of strings defined by specific grammatical rules. Additionally, the theory explores Turing machines, which are abstract representations of computation, and the Church-Turing thesis, which posits that any computation can be performed by a Turing machine. These concepts collectively help in understanding the limits and capabilities of digital computation." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What are the key concepts in the Theory of Digital Computation?", + "response": "The Theory of Digital Computation is like a big toolbox for understanding computers. It has things like algorithms, which are like recipes for solving problems. It also looks at how hard these problems are to solve, called computational complexity. There are also ideas about pretend machines, called automata, and how they work. Formal languages are like special codes that computers understand. Turing machines are like imaginary computers that help us think about what computers can do. The Church-Turing thesis says that these pretend computers can do anything a real computer can do. All these ideas help us know what computers can and can't do.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Digital Computation encompasses several key concepts that form the foundation of computer science. These include algorithms, which are step-by-step procedures for solving problems; computational complexity, which studies the resources required for algorithms to solve problems; automata theory, which examines abstract machines and the problems they can solve; and formal languages, which are sets of strings defined by specific grammatical rules. Additionally, the theory explores Turing machines, which are abstract representations of computation, and the Church-Turing thesis, which posits that any computation can be performed by a Turing machine. These concepts collectively help in understanding the limits and capabilities of digital computation." + }, + "prompt_output": { + "reason": "The response accurately covers all the key concepts mentioned in the reference, including algorithms, computational complexity, automata, formal languages, Turing machines, and the Church-Turing thesis.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Quantum Decoherence and how does it explain the transition from quantum to classical physics?", + "response": "The Theory of Quantum Decoherence is like when a magic trick stops being magical because too many people are watching. In the tiny world of quantum, things can be in many places at once, like magic. But when they mix with the world around them, they start acting normal, like everyday things we see. This helps us understand how the weird quantum world turns into the normal world we know, without needing someone to watch it happen.", + "reference": "The Theory of Quantum Decoherence explains how quantum systems lose their quantum behavior and appear classical when interacting with their environment. It describes the process by which a quantum system's coherent superposition of states becomes a statistical mixture due to entanglement with the environment. This interaction causes the system to lose its quantum coherence, making it behave more like a classical system. Decoherence provides a framework for understanding the apparent collapse of the wave function without invoking a measurement or observer, thus bridging the gap between quantum mechanics and classical physics." + }, + "metric_output": 0, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Quantum Decoherence and how does it explain the transition from quantum to classical physics?", + "response": "The Theory of Quantum Decoherence is like when a magic trick stops being magical because too many people are watching. In the tiny world of quantum, things can be in many places at once, like magic. But when they mix with the world around them, they start acting normal, like everyday things we see. This helps us understand how the weird quantum world turns into the normal world we know, without needing someone to watch it happen.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Quantum Decoherence explains how quantum systems lose their quantum behavior and appear classical when interacting with their environment. It describes the process by which a quantum system's coherent superposition of states becomes a statistical mixture due to entanglement with the environment. This interaction causes the system to lose its quantum coherence, making it behave more like a classical system. Decoherence provides a framework for understanding the apparent collapse of the wave function without invoking a measurement or observer, thus bridging the gap between quantum mechanics and classical physics." + }, + "prompt_output": { + "reason": "The response oversimplifies the concept of Quantum Decoherence and does not accurately capture the technical details provided in the reference. It uses an analogy that may mislead the reader about the actual scientific principles involved.", + "verdict": 0 + }, + "is_accepted": true, + "edited_output": { + "reason": "The response correctly explains the concept of Quantum Decoherence and does factually accurate with the details provided in the reference.", + "verdict": 0 + } + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Hawking Radiation and how does it relate to black holes?", + "response": "Stephen Hawking, a smart scientist, said in 1974 that black holes aren't just big, dark things that suck everything in. He explained that tiny bits of energy, called radiation, can actually come out of them! This happens because little pairs of particles pop up near the edge of the black hole. One gets pulled in, and the other escapes, making it look like the black hole is shining a tiny bit. Over a long time, this can make the black hole shrink and maybe even disappear! This idea was super cool because it mixed two big science ideas: how tiny things work and how big things like stars and planets behave.", + "reference": "The Theory of Hawking Radiation, proposed by physicist Stephen Hawking in 1974, suggests that black holes are not completely black but emit radiation due to quantum effects near the event horizon. This radiation arises because of particle-antiparticle pairs that form near the event horizon. One of the particles falls into the black hole while the other escapes, making it appear as though the black hole is emitting radiation. Over time, this process can lead to the gradual loss of mass and energy from the black hole, potentially causing it to evaporate completely. This theory was groundbreaking as it combined principles of quantum mechanics with general relativity, challenging the notion that nothing can escape from a black hole." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Hawking Radiation and how does it relate to black holes?", + "response": "Stephen Hawking, a smart scientist, said in 1974 that black holes aren't just big, dark things that suck everything in. He explained that tiny bits of energy, called radiation, can actually come out of them! This happens because little pairs of particles pop up near the edge of the black hole. One gets pulled in, and the other escapes, making it look like the black hole is shining a tiny bit. Over a long time, this can make the black hole shrink and maybe even disappear! This idea was super cool because it mixed two big science ideas: how tiny things work and how big things like stars and planets behave.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Hawking Radiation, proposed by physicist Stephen Hawking in 1974, suggests that black holes are not completely black but emit radiation due to quantum effects near the event horizon. This radiation arises because of particle-antiparticle pairs that form near the event horizon. One of the particles falls into the black hole while the other escapes, making it appear as though the black hole is emitting radiation. Over time, this process can lead to the gradual loss of mass and energy from the black hole, potentially causing it to evaporate completely. This theory was groundbreaking as it combined principles of quantum mechanics with general relativity, challenging the notion that nothing can escape from a black hole." + }, + "prompt_output": { + "reason": "The response accurately explains the Theory of Hawking Radiation, including the key concepts of particle-antiparticle pairs, the event horizon, and the implications for black holes, aligning well with the reference.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Special Creation and how does it explain the origin of life?", + "response": "The Theory of Special Creation is like a story that says a magical being made everything, like animals and people, just the way they are now. It says they haven't changed much since they were made. This idea is different from science ideas like evolution, which say that living things have changed a lot over time. People who believe in this theory often read religious books, like the Bible, to understand how everything was made.", + "reference": "The Theory of Special Creation is a religious-based explanation for the origin of life, proposing that life and the universe were created by a supernatural being or deity. This theory suggests that all species were created in their current form and have not undergone significant changes since their creation. It contrasts with scientific theories such as evolution, which propose that life has evolved over time through natural processes. The Theory of Special Creation is often associated with literal interpretations of religious texts, such as the Bible's account of creation in the Book of Genesis." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Special Creation and how does it explain the origin of life?", + "response": "The Theory of Special Creation is like a story that says a magical being made everything, like animals and people, just the way they are now. It says they haven't changed much since they were made. This idea is different from science ideas like evolution, which say that living things have changed a lot over time. People who believe in this theory often read religious books, like the Bible, to understand how everything was made.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Special Creation is a religious-based explanation for the origin of life, proposing that life and the universe were created by a supernatural being or deity. This theory suggests that all species were created in their current form and have not undergone significant changes since their creation. It contrasts with scientific theories such as evolution, which propose that life has evolved over time through natural processes. The Theory of Special Creation is often associated with literal interpretations of religious texts, such as the Bible's account of creation in the Book of Genesis." + }, + "prompt_output": { + "reason": "The response accurately describes the Theory of Special Creation, its contrast with evolution, and its association with religious texts, aligning well with the reference provided.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Multidimensional Space and how does it relate to our understanding of the universe?", + "response": "The Theory of Multidimensional Space is like saying there are more directions to move in than just up, down, and sideways. Scientists think there might be extra hidden spaces we can't see. This idea is used in a science called string theory, which says everything is made of tiny strings that wiggle in these extra spaces. These hidden spaces might help us understand how things like gravity work and how the universe started, like with black holes and the Big Bang!", + "reference": "The Theory of Multidimensional Space suggests that beyond the three dimensions we experience daily, there may exist additional spatial dimensions. This concept is often explored in theoretical physics, particularly in string theory, which posits that the universe is composed of tiny, vibrating strings that exist in multiple dimensions. These extra dimensions could help explain fundamental forces and particles in the universe, potentially unifying gravity with other forces. The theory challenges our perception of reality and opens up possibilities for understanding complex phenomena like black holes and the Big Bang." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Multidimensional Space and how does it relate to our understanding of the universe?", + "response": "The Theory of Multidimensional Space is like saying there are more directions to move in than just up, down, and sideways. Scientists think there might be extra hidden spaces we can't see. This idea is used in a science called string theory, which says everything is made of tiny strings that wiggle in these extra spaces. These hidden spaces might help us understand how things like gravity work and how the universe started, like with black holes and the Big Bang!", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Multidimensional Space suggests that beyond the three dimensions we experience daily, there may exist additional spatial dimensions. This concept is often explored in theoretical physics, particularly in string theory, which posits that the universe is composed of tiny, vibrating strings that exist in multiple dimensions. These extra dimensions could help explain fundamental forces and particles in the universe, potentially unifying gravity with other forces. The theory challenges our perception of reality and opens up possibilities for understanding complex phenomena like black holes and the Big Bang." + }, + "prompt_output": { + "reason": "The response accurately summarizes the Theory of Multidimensional Space and its relation to string theory, black holes, and the Big Bang, aligning well with the reference provided.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Meteor Impacts and how does it explain mass extinctions on Earth?", + "response": "The Theory of Meteor Impacts is like saying a giant space rock crashed into Earth a long time ago and made big changes that caused many animals to disappear. One famous time this happened was about 66 million years ago, and it might have made the dinosaurs go bye-bye. Scientists found a special kind of dirt called iridium that comes from space rocks, which helps them know this happened. When the big rock hit, it made fires and blocked the sun, making it super hard for animals and plants to live.", + "reference": "The Theory of Meteor Impacts suggests that large meteorites or asteroids have collided with Earth in the past, causing significant environmental changes and mass extinctions. One of the most well-known examples is the impact event that occurred approximately 66 million years ago, which is believed to have led to the extinction of the dinosaurs. This theory is supported by evidence such as the presence of a layer of iridium-rich clay in the geological record, which is rare on Earth but common in asteroids. The impact would have caused massive fires, a 'nuclear winter' effect due to dust and aerosols blocking sunlight, and drastic climate changes, leading to the collapse of ecosystems and the extinction of many species." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Meteor Impacts and how does it explain mass extinctions on Earth?", + "response": "The Theory of Meteor Impacts is like saying a giant space rock crashed into Earth a long time ago and made big changes that caused many animals to disappear. One famous time this happened was about 66 million years ago, and it might have made the dinosaurs go bye-bye. Scientists found a special kind of dirt called iridium that comes from space rocks, which helps them know this happened. When the big rock hit, it made fires and blocked the sun, making it super hard for animals and plants to live.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Meteor Impacts suggests that large meteorites or asteroids have collided with Earth in the past, causing significant environmental changes and mass extinctions. One of the most well-known examples is the impact event that occurred approximately 66 million years ago, which is believed to have led to the extinction of the dinosaurs. This theory is supported by evidence such as the presence of a layer of iridium-rich clay in the geological record, which is rare on Earth but common in asteroids. The impact would have caused massive fires, a 'nuclear winter' effect due to dust and aerosols blocking sunlight, and drastic climate changes, leading to the collapse of ecosystems and the extinction of many species." + }, + "prompt_output": { + "reason": "The response accurately summarizes the Theory of Meteor Impacts and its implications for mass extinctions, including the key points about the impact event 66 million years ago, the evidence of iridium, and the environmental consequences.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What are the key principles of Einstein's theory of Special Relativity?", + "response": "Einstein's Special Relativity is like two big ideas. First, no matter how fast you're going, the rules of how things work stay the same for everyone. Second, light always zooms at the same speed, no matter who's watching or how they're moving. Because of these ideas, weird things happen, like time going slower for fast-moving things and things looking squished when they move fast. This changed how we think about space and time, showing they're all mixed together in a big space-time thingy!", + "reference": "Einstein's theory of Special Relativity is based on two key principles. The first is the principle of relativity, which states that the laws of physics are the same for all observers in uniform motion relative to one another. This means that no matter how fast you are moving, the laws of physics apply equally. The second principle is the constancy of the speed of light, which asserts that the speed of light in a vacuum is the same for all observers, regardless of their motion or the motion of the light source. These principles lead to several surprising consequences, such as time dilation, where time appears to pass more slowly for objects moving at high speeds, and length contraction, where objects appear shorter in the direction of motion. Special Relativity fundamentally changed our understanding of space and time, showing that they are interconnected in a four-dimensional spacetime continuum." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What are the key principles of Einstein's theory of Special Relativity?", + "response": "Einstein's Special Relativity is like two big ideas. First, no matter how fast you're going, the rules of how things work stay the same for everyone. Second, light always zooms at the same speed, no matter who's watching or how they're moving. Because of these ideas, weird things happen, like time going slower for fast-moving things and things looking squished when they move fast. This changed how we think about space and time, showing they're all mixed together in a big space-time thingy!", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "Einstein's theory of Special Relativity is based on two key principles. The first is the principle of relativity, which states that the laws of physics are the same for all observers in uniform motion relative to one another. This means that no matter how fast you are moving, the laws of physics apply equally. The second principle is the constancy of the speed of light, which asserts that the speed of light in a vacuum is the same for all observers, regardless of their motion or the motion of the light source. These principles lead to several surprising consequences, such as time dilation, where time appears to pass more slowly for objects moving at high speeds, and length contraction, where objects appear shorter in the direction of motion. Special Relativity fundamentally changed our understanding of space and time, showing that they are interconnected in a four-dimensional spacetime continuum." + }, + "prompt_output": { + "reason": "The response accurately captures the key principles of Einstein's theory of Special Relativity, including the principle of relativity and the constancy of the speed of light, as well as the consequences like time dilation and length contraction.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What are the key principles of quantum mechanics?", + "response": "Quantum mechanics is like a set of rules for tiny things like atoms. One rule is that tiny things can act like both waves and little balls. Another rule says you can't know everything about a tiny thing at the same time. There's also a rule that says tiny things can be in lots of places at once until you look at them. And finally, there's a rule that says tiny things can be connected so that if you change one, the other one changes too, even if they're far apart!", + "reference": "Quantum mechanics is a fundamental theory in physics that describes the physical properties of nature at the scale of atoms and subatomic particles. Key principles include wave-particle duality, which posits that particles can exhibit both wave-like and particle-like properties; the uncertainty principle, which states that certain pairs of physical properties cannot be simultaneously known to arbitrary precision; and superposition, which allows particles to exist in multiple states at once until measured. Additionally, quantum entanglement describes how particles can become interconnected in such a way that the state of one particle instantly influences the state of another, regardless of distance." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What are the key principles of quantum mechanics?", + "response": "Quantum mechanics is like a set of rules for tiny things like atoms. One rule is that tiny things can act like both waves and little balls. Another rule says you can't know everything about a tiny thing at the same time. There's also a rule that says tiny things can be in lots of places at once until you look at them. And finally, there's a rule that says tiny things can be connected so that if you change one, the other one changes too, even if they're far apart!", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "Quantum mechanics is a fundamental theory in physics that describes the physical properties of nature at the scale of atoms and subatomic particles. Key principles include wave-particle duality, which posits that particles can exhibit both wave-like and particle-like properties; the uncertainty principle, which states that certain pairs of physical properties cannot be simultaneously known to arbitrary precision; and superposition, which allows particles to exist in multiple states at once until measured. Additionally, quantum entanglement describes how particles can become interconnected in such a way that the state of one particle instantly influences the state of another, regardless of distance." + }, + "prompt_output": { + "reason": "The response accurately summarizes the key principles of quantum mechanics as outlined in the reference, including wave-particle duality, the uncertainty principle, superposition, and quantum entanglement.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Abiogenesis and how does it explain the origin of life on Earth?", + "response": "The Theory of Abiogenesis is like saying life started from stuff that wasn't alive a long time ago. It means that tiny building blocks of life, like amino acids, could just pop up naturally from simple things. Scientists did experiments, like the Miller-Urey experiment, to show this could happen. This idea helps us understand how life might have started on Earth without magic or anything like that.", + "reference": "The Theory of Abiogenesis posits that life on Earth began from non-living matter through natural processes. It suggests that simple organic compounds formed spontaneously under prebiotic conditions, eventually leading to more complex molecules and the first primitive life forms. This theory is supported by experiments like the Miller-Urey experiment, which demonstrated that amino acids, the building blocks of life, could be synthesized from simple chemicals under conditions thought to resemble those of early Earth. Abiogenesis provides a scientific framework for understanding how life could have originated without the need for supernatural intervention." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Abiogenesis and how does it explain the origin of life on Earth?", + "response": "The Theory of Abiogenesis is like saying life started from stuff that wasn't alive a long time ago. It means that tiny building blocks of life, like amino acids, could just pop up naturally from simple things. Scientists did experiments, like the Miller-Urey experiment, to show this could happen. This idea helps us understand how life might have started on Earth without magic or anything like that.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Abiogenesis posits that life on Earth began from non-living matter through natural processes. It suggests that simple organic compounds formed spontaneously under prebiotic conditions, eventually leading to more complex molecules and the first primitive life forms. This theory is supported by experiments like the Miller-Urey experiment, which demonstrated that amino acids, the building blocks of life, could be synthesized from simple chemicals under conditions thought to resemble those of early Earth. Abiogenesis provides a scientific framework for understanding how life could have originated without the need for supernatural intervention." + }, + "prompt_output": { + "reason": "The response accurately summarizes the Theory of Abiogenesis and includes relevant details such as the formation of simple organic compounds and the Miller-Urey experiment, aligning well with the reference.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the significance of General Relativity in modern physics?", + "response": "General Relativity is a big idea from Albert Einstein in 1915 that changed how we think about gravity. Instead of just pulling things down, gravity is like a big invisible blanket that bends around heavy stuff like stars and planets. This helps us understand cool things like black holes, ripples in space called gravitational waves, and how the universe is getting smaller. Scientists have tested it a lot, like seeing how light bends around big things in space and how planets move just right. It also helps us learn about the Big Bang and how the universe is put together!", + "reference": "General Relativity, formulated by Albert Einstein in 1915, revolutionized our understanding of gravity by describing it as the curvature of spacetime caused by mass and energy. This theory has significant implications in modern physics, providing the framework for understanding phenomena such as black holes, gravitational waves, and the expansion of the universe. It has been confirmed by numerous experiments and observations, including the bending of light around massive objects and the precise orbit of planets. General Relativity also plays a crucial role in the field of cosmology, influencing our understanding of the Big Bang and the overall structure of the universe." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the significance of General Relativity in modern physics?", + "response": "General Relativity is a big idea from Albert Einstein in 1915 that changed how we think about gravity. Instead of just pulling things down, gravity is like a big invisible blanket that bends around heavy stuff like stars and planets. This helps us understand cool things like black holes, ripples in space called gravitational waves, and how the universe is getting smaller. Scientists have tested it a lot, like seeing how light bends around big things in space and how planets move just right. It also helps us learn about the Big Bang and how the universe is put together!", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "General Relativity, formulated by Albert Einstein in 1915, revolutionized our understanding of gravity by describing it as the curvature of spacetime caused by mass and energy. This theory has significant implications in modern physics, providing the framework for understanding phenomena such as black holes, gravitational waves, and the expansion of the universe. It has been confirmed by numerous experiments and observations, including the bending of light around massive objects and the precise orbit of planets. General Relativity also plays a crucial role in the field of cosmology, influencing our understanding of the Big Bang and the overall structure of the universe." + }, + "prompt_output": { + "reason": "The response accurately captures the essence of General Relativity, its implications in modern physics, and mentions key concepts such as black holes, gravitational waves, and the Big Bang, aligning well with the reference.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": { + "reason": "The response accurately captures the essence of General Relativity but it wrongly mentions that the universe is getting smaller instead of larger which is mentioned in the reference.", + "verdict": 0 + } + } + }, + "is_accepted": false + }, + { + "metric_input": { + "user_input": "What are the key principles of the Theory of Group Dynamics?", + "response": "The Theory of Group Dynamics is like figuring out how people in a group play together. It looks at roles, which are like the jobs or parts each person has in the group, and norms, which are the rules everyone follows. It also talks about how much the group members like being together, called cohesion. Plus, it checks out how leaders lead, how decisions are made, and how problems are solved, all to see how well the group works together.", + "reference": "The Theory of Group Dynamics explores how individuals in a group interact and influence each other. Key principles include the concept of roles, which are the expected behaviors of individuals within the group, and norms, which are the shared expectations and rules that guide group behavior. Cohesion, the degree to which group members are attracted to each other and motivated to stay in the group, is also crucial. Additionally, group dynamics examines leadership styles, decision-making processes, and conflict resolution strategies, all of which impact the effectiveness and productivity of the group." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What are the key principles of the Theory of Group Dynamics?", + "response": "The Theory of Group Dynamics is like figuring out how people in a group play together. It looks at roles, which are like the jobs or parts each person has in the group, and norms, which are the rules everyone follows. It also talks about how much the group members like being together, called cohesion. Plus, it checks out how leaders lead, how decisions are made, and how problems are solved, all to see how well the group works together.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Group Dynamics explores how individuals in a group interact and influence each other. Key principles include the concept of roles, which are the expected behaviors of individuals within the group, and norms, which are the shared expectations and rules that guide group behavior. Cohesion, the degree to which group members are attracted to each other and motivated to stay in the group, is also crucial. Additionally, group dynamics examines leadership styles, decision-making processes, and conflict resolution strategies, all of which impact the effectiveness and productivity of the group." + }, + "prompt_output": { + "reason": "The response accurately summarizes the key principles of the Theory of Group Dynamics as outlined in the reference, including roles, norms, cohesion, leadership, decision-making, and problem-solving.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What are the main postulates of the Kinetic Molecular Theory?", + "response": "The Kinetic Molecular Theory is like a set of rules to understand how gases act. It says: 1. Gas is made of lots of tiny bits that move around all the time. 2. These bits are so small that their size doesn't really matter compared to the space they are in. 3. The bits don't push or pull on each other unless they bump into each other, and when they do, they bounce off perfectly. 4. How fast these bits move depends on how hot the gas is. This helps us know how gases behave in different situations.", + "reference": "The Kinetic Molecular Theory explains the behavior of gases and is based on several key postulates: 1. Gases consist of a large number of tiny particles that are in constant, random motion. 2. The volume of the individual gas particles is negligible compared to the volume of the container. 3. Gas particles exert no forces on each other except during collisions, which are perfectly elastic. 4. The average kinetic energy of gas particles is directly proportional to the temperature of the gas in Kelvin. This theory helps in understanding gas laws and the behavior of gases under different conditions." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What are the main postulates of the Kinetic Molecular Theory?", + "response": "The Kinetic Molecular Theory is like a set of rules to understand how gases act. It says: 1. Gas is made of lots of tiny bits that move around all the time. 2. These bits are so small that their size doesn't really matter compared to the space they are in. 3. The bits don't push or pull on each other unless they bump into each other, and when they do, they bounce off perfectly. 4. How fast these bits move depends on how hot the gas is. This helps us know how gases behave in different situations.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Kinetic Molecular Theory explains the behavior of gases and is based on several key postulates: 1. Gases consist of a large number of tiny particles that are in constant, random motion. 2. The volume of the individual gas particles is negligible compared to the volume of the container. 3. Gas particles exert no forces on each other except during collisions, which are perfectly elastic. 4. The average kinetic energy of gas particles is directly proportional to the temperature of the gas in Kelvin. This theory helps in understanding gas laws and the behavior of gases under different conditions." + }, + "prompt_output": { + "reason": "The response accurately summarizes the main postulates of the Kinetic Molecular Theory and aligns well with the reference provided.", + "verdict": 1 + }, + "is_accepted": true, + "edited_output": null + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What are the main principles of the Cell Theory?", + "response": "The Cell Theory is like a big rulebook for understanding living things. It says three important things: 1) Every living thing is made up of tiny building blocks called cells. 2) Cells are the smallest parts that can do all the jobs needed for life. 3) New cells are made when old cells split into two. This helps us know why cells are super important for all life and how life keeps going because cells keep making more cells.", + "reference": "The Cell Theory is a fundamental principle in biology that describes the properties of cells. It consists of three main principles: 1) All living organisms are composed of one or more cells, which are the basic units of life. 2) The cell is the smallest unit of life, and all vital functions of an organism occur within cells. 3) All cells arise from pre-existing cells through the process of cell division. This theory highlights the importance of cells in the structure and function of all living organisms and underscores the continuity of life through cellular reproduction." + }, + "metric_output": 1, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What are the main principles of the Cell Theory?", + "response": "The Cell Theory is like a big rulebook for understanding living things. It says three important things: 1) Every living thing is made up of tiny building blocks called cells. 2) Cells are the smallest parts that can do all the jobs needed for life. 3) New cells are made when old cells split into two. This helps us know why cells are super important for all life and how life keeps going because cells keep making more cells.", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Cell Theory is a fundamental principle in biology that describes the properties of cells. It consists of three main principles: 1) All living organisms are composed of one or more cells, which are the basic units of life. 2) The cell is the smallest unit of life, and all vital functions of an organism occur within cells. 3) All cells arise from pre-existing cells through the process of cell division. This theory highlights the importance of cells in the structure and function of all living organisms and underscores the continuity of life through cellular reproduction." + }, + "prompt_output": { + "reason": "The response does not accurately reflect the reference. While it captures the essence of the Cell Theory, it uses informal language and omits key details such as the emphasis on cells being the basic units of life and the process of cell division.", + "verdict": 0 + }, + "is_accepted": true, + "edited_output": { + "reason": "The response does provides a simplified and accurate representation of the reference", + "verdict": 1 + } + } + }, + "is_accepted": true + }, + { + "metric_input": { + "user_input": "What is the Theory of Nitrogen Fixation and how does it impact agriculture?", + "response": "The Theory of Nitrogen Fixation is like a magic trick where tiny helpers called bacteria turn air into food for plants. These bacteria live in little bumps on the roots of plants like corn and wheat. This magic helps plants grow better without needing a lot of extra plant food from factories, making farming kinder to the earth and helping farmers grow more food!", + "reference": "The Theory of Nitrogen Fixation explains the process by which certain organisms convert atmospheric nitrogen into a form that plants can use for growth. This process is primarily carried out by symbiotic bacteria, such as Rhizobium, which form nodules on the roots of leguminous plants. Nitrogen fixation is crucial for agriculture as it enriches the soil with nitrogen, reducing the need for chemical fertilizers and promoting sustainable farming practices. This natural process helps maintain soil fertility and supports the growth of crops, leading to increased agricultural productivity." + }, + "metric_output": 0, + "prompts": { + "single_turn_aspect_critic_prompt": { + "prompt_input": { + "user_input": "What is the Theory of Nitrogen Fixation and how does it impact agriculture?", + "response": "The Theory of Nitrogen Fixation is like a magic trick where tiny helpers called bacteria turn air into food for plants. These bacteria live in little bumps on the roots of plants like corn and wheat. This magic helps plants grow better without needing a lot of extra plant food from factories, making farming kinder to the earth and helping farmers grow more food!", + "retrieved_contexts": null, + "reference_contexts": null, + "reference": "The Theory of Nitrogen Fixation explains the process by which certain organisms convert atmospheric nitrogen into a form that plants can use for growth. This process is primarily carried out by symbiotic bacteria, such as Rhizobium, which form nodules on the roots of leguminous plants. Nitrogen fixation is crucial for agriculture as it enriches the soil with nitrogen, reducing the need for chemical fertilizers and promoting sustainable farming practices. This natural process helps maintain soil fertility and supports the growth of crops, leading to increased agricultural productivity." + }, + "prompt_output": { + "reason": "The response oversimplifies the concept of nitrogen fixation and does not accurately convey the scientific details provided in the reference. It lacks specific information about the organisms involved and the importance of nitrogen fixation in agriculture.", + "verdict": 0 + }, + "is_accepted": true, + "edited_output": { + "reason": "The response mentions air instead of nitrogen which is wrong.", + "verdict": 0 + } + } + }, + "is_accepted": true + } + ] +} \ No newline at end of file diff --git a/docs/_static/imgs/after_training.png b/docs/_static/imgs/after_training.png new file mode 100644 index 000000000..372b16c31 Binary files /dev/null and b/docs/_static/imgs/after_training.png differ diff --git a/docs/_static/imgs/annotation.png b/docs/_static/imgs/annotation.png new file mode 100644 index 000000000..1e742d755 Binary files /dev/null and b/docs/_static/imgs/annotation.png differ diff --git a/docs/_static/imgs/before_training.png b/docs/_static/imgs/before_training.png new file mode 100644 index 000000000..74dbba25a Binary files /dev/null and b/docs/_static/imgs/before_training.png differ diff --git a/docs/howtos/customizations/index.md b/docs/howtos/customizations/index.md index b2aabcb87..5e71b59cc 100644 --- a/docs/howtos/customizations/index.md +++ b/docs/howtos/customizations/index.md @@ -12,6 +12,7 @@ How to customize various aspects of Ragas to suit your needs. - [Write your own metrics](./metrics/_write_your_own_metric.md) - [Adapt metrics to target language](./metrics/_metrics_language_adaptation.md) - [Trace evaluations with Observability tools](metrics/tracing.md) +- [Train and align metric](./metrics/_train_your_own_metric.md) ## Testset Generation diff --git a/docs/howtos/customizations/metrics/_train_your_own_metric.md b/docs/howtos/customizations/metrics/_train_your_own_metric.md new file mode 100644 index 000000000..27957bf43 --- /dev/null +++ b/docs/howtos/customizations/metrics/_train_your_own_metric.md @@ -0,0 +1,181 @@ +# Train and Align your own Metric + +LLM as judge metric often makes mistakes and lack alignment with human evaluators. This makes them risky to use as their results cannot be trusted fully. Now, you can fix this using ragas. This simple tutorial notebook showcasing how to train and align any LLM as judge metric using ragas. One can use this to train any LLM based metric in ragas. + + +## Import required modules + + +```python +import os +from datasets import load_dataset +from ragas import evaluate, EvaluationDataset +from ragas.metrics import AspectCritic + +``` + +### Setup the models used for evaluation and training +You may choose any LLM model for training and evaluation. Here's [how to do it](../customize_models.md) + + +```python +from ragas.llms import LangchainLLMWrapper +from ragas.embeddings import LangchainEmbeddingsWrapper +from langchain_openai import ChatOpenAI +from langchain_openai import OpenAIEmbeddings +llm = LangchainLLMWrapper(ChatOpenAI(model="gpt-4o-mini")) +embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings()) +``` + +## Evaluation + +### Load sample evaluation dataset +Here, we are loading the sample dataset for evaluation. You can replace it with your own dataset. + + +```python +dataset = load_dataset("explodinggradients/ELI5",split="test") +eval_dataset = EvaluationDataset.from_hf_dataset(dataset) +print(dataset[10]) +``` + + +``` +{ + "user_input": "What is the Theory of Cosmic Inflation and how does it explain the early universe?", + "reference": "The Theory of Cosmic Inflation proposes that the universe underwent an exponential expansion in its earliest moments, just after the Big Bang. This rapid expansion helps to explain several observed phenomena in cosmology, such as the uniformity of the cosmic microwave background radiation and the large-scale structure of the universe. Inflation theory suggests that tiny quantum fluctuations were stretched to macroscopic scales, seeding the formation of galaxies and other cosmic structures. It also addresses the horizon and flatness problems, providing a more comprehensive understanding of the universe's initial conditions.", + "response": "The Theory of Cosmic Inflation is like saying the universe blew up like a giant balloon really fast right after it was born. This helps us understand why the universe looks the same everywhere we look and how galaxies and stars started to form. It also helps answer some big questions about why the universe is so flat and even." +} +``` + + +The dataset contains user input, reference and response. The metric will evaluate the response based on the reference. The response here is in ELI5 format, which is a simple way of explaining complex topics. This is a good example to align the metric with human evaluators as in this situation the human evaluator will consider the response as correct if the ELI5 response is accurate and complete compared to the reference and incorrect if the response contains any factual inaccuracy. + +### Setup the Metric +You may use any LLM based metric. For simplicity, I am using aspect critic metric and setting it up so that it can compare the response with the reference. + + +```python +critic = AspectCritic(name="answer_correctness",definition="Given the user_input, reference and response. Is the response correct compared with the reference",llm=llm) + +``` + +### Evaluate + + +```python +results = evaluate(eval_dataset,metrics=[critic]) + +``` + + Evaluating: 100%|██████████████████████████████████████████████████████████| 56/56 [00:01<00:00, 3.49it/s] + + + +## Review and Annotate + +Now you have the evaluation results. Now it's time to review the evaluations and give feedback to the metric. This feedback will be used to train the metric. For this you can use [app.ragas](https://app.ragas.io) or any other annotation tool like prodigy, label studio etc. + +If you're using app.ragas, + +- Go to [app.ragas](https://app.ragas.io) and login. +- Then go to [App tokens](https://app.ragas.io/dashboard/settings/app-tokens) and create a new app token. +- Set the token in the environment variable + +```python +os.environ['RAGAS_APP_TOKEN'] = 'your_app_token' +``` + +Once that's done, you can upload the evaluation results to app.ragas using the following code. + +```python +results.upload() +``` + +Evaluation results uploaded! View at https://app.ragas.io/dashboard/alignment/evaluation/a6baf6ff-027f-4097-89e3-e11c70b8cf61 +'https://app.ragas.io/dashboard/alignment/evaluation/a6baf6ff-027f-4097-89e3-e11c70b8cf61' + + + +### Review and annotate some results +You may now view and annotate the evaluation results in app.ragas. These annotations will be used to train the metric. Please make sure to annotate at least 15-20 examples for good results. + +
+ ![Annotating](../../../_static/imgs/annotation.png){width="600"} +
Annotating results
+
+ +Here is a sample annotation for the above example. You can [download](../../../_static/edited_chain_runs.json) and use it. + +## Training and Alignment + +### Train the metric +Download the annotated samples from app.ragas using `Download annotated json` button. +Instruction and demonstration configurations are required tells ragas how to optimize instruction and few shot demonstrations respectively. You can customize these configurations as per your requirements. + +```python +from ragas.config import InstructionConfig,DemonstrationConfig +demo_config = DemonstrationConfig(embedding = embeddings) +inst_config = InstructionConfig(llm=llm) +``` + + +```python +critic.train(path="edited_chain_runs.json",demonstration_config=demo_config,instruction_config=inst_config) +``` + + Fitness Evaluation Step 4/4: 100%|█████████████████████████████████████| 146/146 [00:24<00:00, 6.03it/s] + Few-shot examples [single_turn_aspect_critic_prompt]: 100%|██████████████| 18/18 [00:09<00:00, 1.82it/s] + + +### Analysis +Now, let's do some analysis on the trained metric. + +First, let's take a look at new instructions that was obtained for the metric after training. + + +```python +print(critic.get_prompts()['single_turn_aspect_critic_prompt'].instruction) +``` + +``` +Evaluate the provided user responses against the reference information for accuracy and completeness. +Assign a verdict of 1 if the response is accurate and aligns well with the reference, or 0 if it contains inaccuracies or misrepresentations. +``` + +compared with old prompt +``` +Evaluate the Input based on the criterial defined. Use only 'Yes' (1) and 'No' (0) as verdict. +Criteria Definition: Given the user_input, reference and response. Is the response correct compared with the reference +``` + + +### Re-evaluate +Let's evaluate again and see if the metric has improved for any un-annotated examples. + + +```python +results = evaluate(eval_dataset,metrics=[critic]) + +``` + + Evaluating: 100%|████████████████████████████████████████████████████████| 56/56 [00:28<00:00, 1.78it/s] + + + +```python +results.upload() +``` + +Evaluation results uploaded! View at https://app.ragas.io/dashboard/alignment/evaluation/687e7cdf-ff31-4c15-9780-c179207c929c +'https://app.ragas.io/dashboard/alignment/evaluation/687e7cdf-ff31-4c15-9780-c179207c929c' + +### Inspect and compare results + +Go to [app.ragas](https://app.ragas.io/dashboard) dashboard and compare the results before and after training. + +Here in this case, the metric has improved significantly. You can see the difference in the scores. To show the difference, let's compares the scores and changed reasoning for one specific example before and after training. + +| ![Image 1](../../../_static/imgs/before_training.png) | ![Image 2](../../../_static/imgs/after_training.png) | +|:-------------------------------:|:-------------------------------:| +| *Before training* | *After training* | \ No newline at end of file diff --git a/docs/howtos/customizations/metrics/train_your_own_metric.ipynb b/docs/howtos/customizations/metrics/train_your_own_metric.ipynb new file mode 100644 index 000000000..8eeebdd83 --- /dev/null +++ b/docs/howtos/customizations/metrics/train_your_own_metric.ipynb @@ -0,0 +1,364 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "ff2c00f7", + "metadata": { + "notebookRunGroups": { + "groupValue": "2" + } + }, + "source": [ + "## Train your own metric\n", + "\n", + "LLM as judge metric often makes mistakes and lack alignment with human evaluators. This makes them risky to use as their results cannot be trusted fully. Now, you can fix this using ragas. This simple tutorial notebook showcasing how to train and align any LLM as judge metric using ragas. One can use this to train any LLM based metric in ragas. \n", + "\n" + ] + }, + { + "cell_type": "markdown", + "id": "53d79c6c-5316-41ce-84d1-61a7a3d4a320", + "metadata": {}, + "source": [ + "\n", + "### Import required modules" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b0f9dd63-91c7-4ba4-a21f-7dbb54ce2414", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/requests/__init__.py:102: RequestsDependencyWarning: urllib3 (1.26.20) or chardet (5.2.0)/charset_normalizer (None) doesn't match a supported version!\n", + " warnings.warn(\"urllib3 ({}) or chardet ({})/charset_normalizer ({}) doesn't match a supported \"\n", + "/opt/homebrew/Caskroom/miniforge/base/envs/ragas/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "import os\n", + "from datasets import load_dataset\n", + "from ragas import evaluate, EvaluationDataset\n", + "from ragas.metrics import AspectCritic\n" + ] + }, + { + "cell_type": "markdown", + "id": "d1d03eb0", + "metadata": {}, + "source": [ + "Now, sign up for a free account at [app.ragas](https://app.ragas.io) and get your API key.\n", + "Navigate to App tokens -> Create new token. Copy the key and paste it in the below code. Store it safely." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d83b4505-68f4-4a07-8f79-73da8d4ef3f7", + "metadata": {}, + "outputs": [], + "source": [ + "os.environ['RAGAS_APP_TOKEN'] = 'your_app_token'" + ] + }, + { + "cell_type": "markdown", + "id": "3d82272d-eb13-483c-ba06-db4c0a73f3fd", + "metadata": {}, + "source": [ + "### Setup the models used for evaluation and training\n", + "You may choose any LLM model for training and evaluation. Here's [how to do it](../customize_models.md)" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "2461866d-e8c6-4cbf-b1f2-20079069c1ff", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.llms import LangchainLLMWrapper\n", + "from ragas.embeddings import LangchainEmbeddingsWrapper\n", + "from langchain_openai import ChatOpenAI\n", + "from langchain_openai import OpenAIEmbeddings\n", + "llm = LangchainLLMWrapper(ChatOpenAI(model=\"gpt-4o-mini\"))\n", + "embeddings = LangchainEmbeddingsWrapper(OpenAIEmbeddings())" + ] + }, + { + "cell_type": "markdown", + "id": "6ebeb666", + "metadata": {}, + "source": [ + "### Load sample evaluation dataset\n", + "Here, we are loading the sample dataset for evaluation. You can replace it with your own dataset. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a9dcab3f-325a-4b55-a662-759db6a2a9f1", + "metadata": {}, + "outputs": [], + "source": [ + "dataset = load_dataset(\"explodinggradients/ELI5\",split=\"test\")\n", + "eval_dataset = EvaluationDataset.from_hf_dataset(dataset)" + ] + }, + { + "cell_type": "markdown", + "id": "64a83029", + "metadata": {}, + "source": [ + "### Setup the Metric\n", + "You may use any LLM based metric. For simplicity, I am using aspect critic metric and setting it up so that it can compare the response with the reference." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "43df046e-07de-4967-a232-54578fb3f880", + "metadata": {}, + "outputs": [], + "source": [ + "critic = AspectCritic(name=\"answer_correctness\",definition=\"Given the user_input, reference and response. Is the response correct compared with the reference\",llm=llm)\n" + ] + }, + { + "cell_type": "markdown", + "id": "27caa5a5", + "metadata": {}, + "source": [ + "### Evaluate and Upload the results" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a6964abe-0360-4249-946a-5c388a166758", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating: 100%|██████████████████████████████████████████████████████████| 5/5 [00:01<00:00, 3.49it/s]\n" + ] + } + ], + "source": [ + "results = evaluate(eval_dataset,metrics=[critic])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "ea52de01-b639-4011-8def-6321b54c2bf3", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluation results uploaded! View at https://app.ragas.io/dashboard/alignment/evaluation/a6baf6ff-027f-4097-89e3-e11c70b8cf61\n" + ] + }, + { + "data": { + "text/plain": [ + "'https://app.ragas.io/dashboard/alignment/evaluation/a6baf6ff-027f-4097-89e3-e11c70b8cf61'" + ] + }, + "execution_count": 17, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results.upload()" + ] + }, + { + "cell_type": "markdown", + "id": "99767b85-08cd-45d7-b81b-5ab2c6aeffd3", + "metadata": {}, + "source": [ + "### Review and annotate some results\n", + "You may now view and annotate the evaluation results in app.ragas. These annotations will be used to train the metric. Please make sure to annotate at least 15-20 examples for good results." + ] + }, + { + "cell_type": "markdown", + "id": "328a2463-08f6-4639-b63f-2e27966afd12", + "metadata": {}, + "source": [ + "### Train the metric\n", + "Download the annotated samples from app.ragas using `Download annotated json` button. " + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "5fb47d6f-4514-4dc3-89ca-0be074f58397", + "metadata": {}, + "outputs": [], + "source": [ + "from ragas.config import InstructionConfig,DemonstrationConfig\n", + "demo_config = DemonstrationConfig(embedding = embeddings)\n", + "inst_config = InstructionConfig(llm=llm)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d2df934d-5701-4c64-972f-2455d3d915d7", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Feedback Mutation Step 2/4: 19%|███████▍ | 28/146 [00:03<00:10, 11.23it/s]No samples found for the feedback generation.\n", + "No feedbacks found for the prompt single_turn_aspect_critic_prompt. Returning the original prompt.\n", + "Feedback Mutation Step 2/4: 23%|████████▊ | 33/146 [00:06<00:26, 4.33it/s]No samples found for the feedback generation.\n", + "No feedbacks found for the prompt single_turn_aspect_critic_prompt. Returning the original prompt.\n", + "Feedback Mutation Step 2/4: 24%|█████████▎ | 35/146 [00:06<00:22, 4.97it/s]Error in LangChainTracer.on_chain_end callback: TracerException('No indexed run ID 2046bdfe-27cc-4ce2-b999-3e8fc674969c.')\n", + "Fitness Evaluation Step 4/4: 100%|█████████████████████████████████████| 146/146 [00:24<00:00, 6.03it/s]\n", + "Few-shot examples [single_turn_aspect_critic_prompt]: 100%|██████████████| 18/18 [00:09<00:00, 1.82it/s]\n" + ] + } + ], + "source": [ + "critic.train(path=\"edited_chain_runs.json\",demonstration_config=demo_config,instruction_config=inst_config)" + ] + }, + { + "cell_type": "markdown", + "id": "7ea9e387-e413-42c2-a188-bce106a7d526", + "metadata": {}, + "source": [ + "### Inspect\n", + "Now, let's do some analysis on the trained metric.\n", + "\n", + "First, let's take a look at new instructions that was obtained for the metric after training." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "74574d75-dc1d-4319-99db-ea7ac284b098", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'Evaluate the provided user responses against the reference information for accuracy and completeness. Assign a verdict of 1 if the response is accurate and aligns well with the reference, or 0 if it contains inaccuracies or misrepresentations.'" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "critic.get_prompts()['single_turn_aspect_critic_prompt'].instruction" + ] + }, + { + "cell_type": "markdown", + "id": "3d710de9-9cb2-4517-a816-11d52e41c35e", + "metadata": {}, + "source": [ + "#### Re-evaluate\n", + "Let's evaluate again and see if the metric has improved for any un-annotated examples." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "c6beed95-63d6-4df5-afe4-09c4a1c5f1e4", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating: 100%|████████████████████████████████████████████████████████| 50/50 [00:28<00:00, 1.78it/s]\n" + ] + } + ], + "source": [ + "results = evaluate(eval_dataset,metrics=[critic])\n" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "8d125209-fe42-4299-b878-fefc0d837247", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Evaluation results uploaded! View at https://app.ragas.io/dashboard/alignment/evaluation/687e7cdf-ff31-4c15-9780-c179207c929c\n" + ] + }, + { + "data": { + "text/plain": [ + "'https://app.ragas.io/dashboard/alignment/evaluation/687e7cdf-ff31-4c15-9780-c179207c929c'" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "results.upload()" + ] + }, + { + "cell_type": "markdown", + "id": "8fc991be", + "metadata": {}, + "source": [ + "Here in my case, the metric has improved significantly. You can see the difference in the scores. To show the difference, let's compares the scores and changed reasoning for one specific example before and after training." + ] + }, + { + "cell_type": "markdown", + "id": "e09b041f", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "ragas", + "language": "python", + "name": "ragas" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.20" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/mkdocs.yml b/mkdocs.yml index 728b12304..88c919b1d 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -82,6 +82,7 @@ nav: - Adapt Metrics to Languages: howtos/customizations/metrics/_metrics_language_adaptation.md - Write your own Metrics: howtos/customizations/metrics/_write_your_own_metric.md - Write your own Metrics - (advanced): howtos/customizations/metrics/_write_your_own_metric_advanced.md + - Train and Align Metrics: howtos/customizations/metrics/_train_your_own_metric.md - Testset Generation: - Non-English Testset Generation: howtos/customizations/testgenerator/_language_adaptation.md - Persona Generation: howtos/customizations/testgenerator/_persona_generator.md