Evals

evals/ComplexExtraction/report.txt

@@ -3,7 +3,7 @@ Acme Insurance project to implement SalesTech CRM solution is currently
 in RED status due to delayed delivery of document production system, led
 by 3rd party vendor - Alfatech. Customer (Acme) is discussing the resolution
 with the vendor. Due to dependencies it will result in delay of the
-ecommerce track by 2 sprints. System integrator (SysCorp) are working
+ecommerce track by 2 sprints. System integrator (SysCorp) is working
 to absorb some of the delay by deploying extra resources to speed up
 development when the doc production is done. Another issue is that the
 customer is not able to provide the test data for the ecommerce track.

evals/ComplexExtraction/run.php

@@ -1,60 +1,70 @@
 <?php
 
+use Cognesy\Instructor\Enums\Mode;
 use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExperiment;
 use Cognesy\Instructor\Extras\Evals\Contracts\Metric;
+use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
+use Cognesy\Instructor\Extras\Evals\Data\Feedback;
 use Cognesy\Instructor\Extras\Evals\Data\InferenceCases;
 use Cognesy\Instructor\Extras\Evals\Data\InstructorData;
 use Cognesy\Instructor\Extras\Evals\Experiment;
 use Cognesy\Instructor\Extras\Evals\Inference\RunInstructor;
-use Cognesy\Instructor\Extras\Evals\Metrics\BooleanCorrectness;
 use Cognesy\Instructor\Extras\Evals\ExperimentSuite;
+use Cognesy\Instructor\Extras\Evals\Metrics\PercentageCorrectness;
+use Cognesy\Instructor\Extras\Sequence\Sequence;
+use Cognesy\Instructor\Features\LLM\Data\Usage;
 
 $loader = require 'vendor/autoload.php';
 $loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
 
-$cases = InferenceCases::get(
-    connections: [],
-    modes: [],
-    stream: []
-);
-
 class Company {
     public string $name;
     public int $foundingYear;
 }
 
+class CompanyEval implements CanEvaluateExperiment
+{
+    public array $expectations;
+
+    public function __construct(array $expectations) {
+        $this->expectations = $expectations;
+    }
+
+    public function evaluate(Experiment $experiment) : Evaluation {
+        $expectedEvents = $this->expectations['events'];
+        /** @var Sequence $events */
+        $events = $experiment->response->value();
+        $result = ($expectedEvents - count($events->list)) / $expectedEvents;
+        return new Evaluation(
+            metric: new PercentageCorrectness('found', $result),
+            feedback: Feedback::none(),
+            usage: Usage::none(),
+        );
+    }
+}
+
+$report = file_get_contents(__DIR__ . '/report.txt');
+$examples = require 'examples.php';
+$prompt = 'Extract a list of project events with all the details from the provided input in JSON format using schema: <|json_schema|>';
+$responseModel = Sequence::of(ProjectEvent::class);
+
 $data = new InstructorData(
     messages: [
         ['role' => 'user', 'content' => 'YOUR GOAL: Use tools to store the information from context based on user questions.'],
         ['role' => 'user', 'content' => 'CONTEXT: Our company ACME was founded in 2020.'],
-        //['role' => 'user', 'content' => 'EXAMPLE CONTEXT: Sony was established in 1946 by Akio Morita.'],
-        //['role' => 'user', 'content' => 'EXAMPLE RESPONSE: ```json{"name":"Sony","year":1899}```'],
         ['role' => 'user', 'content' => 'What is the name and founding year of our company?'],
     ],
     responseModel: Company::class,
 );
 
-class CompanyEval implements CanEvaluateExperiment
-{
-    public function evaluate(Experiment $experiment) : Metric {
-        /** @var Person $decoded */
-        $person = $experiment->response->value();
-        $result = $person->name === 'ACME'
-            && $person->foundingYear === 2020;
-        return new BooleanCorrectness($result);
-    }
-}
-
-//Debug::enable();
-
-//$report = file_get_contents(__DIR__ . '/report.txt');
-//$examples = require 'examples.php';
-//$prompt = 'Extract a list of project events with all the details from the provided input in JSON format using schema: <|json_schema|>';
-//$responseModel = Sequence::of(ProjectEvent::class);
-
 $runner = new ExperimentSuite(
+    cases: InferenceCases::only(
+        connections: ['openai', 'anthropic', 'gemini', 'cohere'],
+        modes: [Mode::Tools],
+        stream: [true, false]
+    ),
     executor: new RunInstructor($data),
-    evaluator: new CompanyEval(),
+    evaluators: new CompanyEval(expectations: ['events' => 12]),
 );
 
-$outputs = $runner->execute($cases);
+$outputs = $runner->execute();

evals/LLMModes/CompanyEval.php

@@ -4,9 +4,11 @@
 
 use Cognesy\Instructor\Enums\Mode;
 use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExperiment;
+use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
+use Cognesy\Instructor\Extras\Evals\Data\Feedback;
 use Cognesy\Instructor\Extras\Evals\Experiment;
 use Cognesy\Instructor\Extras\Evals\Metrics\BooleanCorrectness;
-use Cognesy\Instructor\Extras\Evals\Contracts\Metric;
+use Cognesy\Instructor\Features\LLM\Data\Usage;
 use Cognesy\Instructor\Utils\Str;
 
 class CompanyEval implements CanEvaluateExperiment
@@ -17,13 +19,17 @@ public function __construct(array $expectations) {
         $this->expectations = $expectations;
     }
 
-    public function evaluate(Experiment $experiment) : Metric {
+    public function evaluate(Experiment $experiment) : Evaluation {
         $isCorrect = match ($experiment->mode) {
             Mode::Text => $this->validateText($experiment),
             Mode::Tools => $this->validateToolsData($experiment),
             default => $this->validateDefault($experiment),
         };
-        return new BooleanCorrectness($isCorrect);
+        return new Evaluation(
+            metric: new BooleanCorrectness('is_correct', $isCorrect),
+            feedback: Feedback::none(),
+            usage: Usage::none(),
+        );
     }
 
     private function validateToolsData(Experiment $experiment) : bool {

evals/LLMModes/run.php

@@ -3,19 +3,15 @@
 $loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
 $loader->add('Cognesy\\Evals\\', __DIR__ . '../../evals/');
 
+use Cognesy\Instructor\Extras\Evals\Aggregators\FirstMetric;
+use Cognesy\Instructor\Extras\Evals\Aggregators\SelectedMetric;
 use Cognesy\Instructor\Extras\Evals\Data\InferenceCases;
 use Cognesy\Instructor\Extras\Evals\Data\InferenceData;
 use Cognesy\Instructor\Extras\Evals\Data\InferenceSchema;
 use Cognesy\Instructor\Extras\Evals\Inference\RunInference;
 use Cognesy\Instructor\Extras\Evals\ExperimentSuite;
 use Cognesy\Evals\LLMModes\CompanyEval;
 
-$cases = InferenceCases::except(
-    connections: [],
-    modes: [],
-    stream: []
-);
-
 $data = new InferenceData(
     messages: [
         ['role' => 'user', 'content' => 'YOUR GOAL: Use tools to store the information from context based on user questions.'],
@@ -46,13 +42,18 @@
     ),
 );
 
-$runner = new ExperimentSuite(
+$experiments = new ExperimentSuite(
+    cases: InferenceCases::except(
+        connections: [],
+        modes: [],
+        stream: [],
+    ),
     executor: new RunInference($data),
-    evaluator: new CompanyEval(expectations: [
+    evaluators: new CompanyEval(expectations: [
         'name' => 'ACME',
         'foundingYear' => 2020
     ]),
-    cases: $cases,
+    aggregator: new FirstMetric()
 );
 
-$outputs = $runner->execute();
+$outputs = $experiments->execute();

evals/SimpleExtraction/CompanyEval.php

@@ -3,9 +3,11 @@
 namespace Cognesy\Evals\SimpleExtraction;
 
 use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExperiment;
+use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
+use Cognesy\Instructor\Extras\Evals\Data\Feedback;
 use Cognesy\Instructor\Extras\Evals\Experiment;
 use Cognesy\Instructor\Extras\Evals\Metrics\BooleanCorrectness;
-use Cognesy\Instructor\Extras\Evals\Contracts\Metric;
+use Cognesy\Instructor\Features\LLM\Data\Usage;
 
 class CompanyEval implements CanEvaluateExperiment
 {
@@ -15,10 +17,14 @@ public function __construct(array $expectations) {
         $this->expectations = $expectations;
     }
 
-    public function evaluate(Experiment $experiment) : Metric {
+    public function evaluate(Experiment $experiment) : Evaluation {
         $company = $experiment->response->value();
         $isCorrect = $company->name === $this->expectations['name']
             && $company->foundingYear === $this->expectations['foundingYear'];
-        return new BooleanCorrectness($isCorrect);
+        return new Evaluation(
+            metric: new BooleanCorrectness('is_correct', $isCorrect),
+            feedback: Feedback::none(),
+            usage: Usage::none(),
+        );
     }
 }

evals/SimpleExtraction/run.php

@@ -1,6 +1,7 @@
 <?php
 
 use Cognesy\Instructor\Enums\Mode;
+use Cognesy\Instructor\Extras\Evals\Aggregators\FirstMetric;
 use Cognesy\Instructor\Extras\Evals\Data\InferenceCases;
 use Cognesy\Instructor\Extras\Evals\Data\InstructorData;
 use Cognesy\Instructor\Extras\Evals\Inference\RunInstructor;
@@ -11,30 +12,27 @@
 $loader = require 'vendor/autoload.php';
 $loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
 
-$cases = InferenceCases::except(
-    connections: [],
-    modes: [Mode::Text],
-    stream: []
-);
-
 $data = new InstructorData(
     messages: [
         ['role' => 'user', 'content' => 'YOUR GOAL: Use tools to store the information from context based on user questions.'],
         ['role' => 'user', 'content' => 'CONTEXT: Our company ACME was founded in 2020.'],
-        //['role' => 'user', 'content' => 'EXAMPLE CONTEXT: Sony was established in 1946 by Akio Morita.'],
-        //['role' => 'user', 'content' => 'EXAMPLE RESPONSE: ```json{"name":"Sony","year":1899}```'],
         ['role' => 'user', 'content' => 'What is the name and founding year of our company?'],
     ],
     responseModel: Company::class,
 );
 
 $runner = new ExperimentSuite(
+    cases: InferenceCases::except(
+        connections: ['ollama'],
+        modes: [Mode::Text],
+        stream: []
+    ),
     executor: new RunInstructor($data),
-    evaluator: new CompanyEval(expectations: [
+    evaluators: new CompanyEval(expectations: [
         'name' => 'ACME',
         'foundingYear' => 2020
     ]),
-    cases: $cases,
+    aggregator: new FirstMetric()
 );
 
 $outputs = $runner->execute();

src/Extras/Evals/Aggregators/FirstMetric.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace Cognesy\Instructor\Extras\Evals\Aggregators;
+
+use Cognesy\Instructor\Extras\Evals\Contracts\CanAggregateValues;
+use Cognesy\Instructor\Extras\Evals\Contracts\Metric;
+use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Metrics\NullMetric;
+
+class FirstMetric implements CanAggregateValues
+{
+    public function aggregate(Experiment $experiment): Metric {
+        $firstEval = $experiment->evaluations[0] ?? null;
+        return $firstEval ? $firstEval->metric : new NullMetric();
+    }
+}

src/Extras/Evals/Aggregators/SelectedMetric.php

@@ -0,0 +1,24 @@
+<?php
+
+namespace Cognesy\Instructor\Extras\Evals\Aggregators;
+
+use Cognesy\Instructor\Extras\Evals\Contracts\CanAggregateValues;
+use Cognesy\Instructor\Extras\Evals\Contracts\Metric;
+use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Metrics\NullMetric;
+
+class SelectedMetric implements CanAggregateValues
+{
+    public function __construct(
+        private string $name
+    ) {}
+
+    public function aggregate(Experiment $experiment): Metric {
+        foreach ($experiment->evaluations as $evaluation) {
+            if ($evaluation->metric->name() === $this->name) {
+                return $evaluation->metric;
+            }
+        }
+        return new NullMetric();
+    }
+}

src/Extras/Evals/Console/Display.php

@@ -26,7 +26,7 @@ public function before(Experiment $experiment) : void {
     public function after(Experiment $experiment) : void {
         $answer = $experiment->notes;
         $answerLine = str_replace("\n", '\n', $answer);
-        $metric = $experiment->metric;
+        $value = $experiment->aggregate;
         $timeElapsed = $experiment->timeElapsed;
         $tokensPerSec = $experiment->outputTps();
         $exception = $experiment->exception;
@@ -45,7 +45,7 @@ public function after(Experiment $experiment) : void {
             echo Console::columns([
                 [9, $this->timeFormat($timeElapsed), STR_PAD_LEFT, [Color::DARK_YELLOW]],
                 [10, $this->tokensPerSecFormat($tokensPerSec), STR_PAD_LEFT, [Color::CYAN]],
-                [6, $metric->toString(), STR_PAD_BOTH, $metric->toCliColor()],
+                [6, $value->toString(), STR_PAD_BOTH, $value->toCliColor()],
                 [60, $answerLine, STR_PAD_RIGHT, [Color::WHITE, Color::BG_BLACK]],
             ], 120);
         }

src/Extras/Evals/Contracts/CanAggregateValues.php

@@ -0,0 +1,16 @@
+<?php
+
+namespace Cognesy\Instructor\Extras\Evals\Contracts;
+
+use Cognesy\Instructor\Extras\Evals\Experiment;
+
+interface CanAggregateValues
+{
+    /**
+     * Aggregate the given values into a single metric.
+     *
+     * @param Experiment $experiment
+     * @return Metric
+     */
+    public function aggregate(Experiment $experiment): Metric;
+}

src/Extras/Evals/Contracts/CanEvaluateExperiment.php

@@ -2,9 +2,10 @@
 
 namespace Cognesy\Instructor\Extras\Evals\Contracts;
 
+use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
 use Cognesy\Instructor\Extras\Evals\Experiment;
 
 interface CanEvaluateExperiment
 {
-    public function evaluate(Experiment $experiment) : Metric;
+    public function evaluate(Experiment $experiment) : Evaluation;
 }

src/Extras/Evals/Contracts/Metric.php

@@ -4,7 +4,9 @@
 
 interface Metric
 {
+    public function name() : string;
     public function value() : mixed;
     public function toLoss() : float;
     public function toScore() : float;
-}
+    public function toString () : string;
+}

src/Extras/Evals/Data/BooleanCorrectnessAnalysis.php

@@ -0,0 +1,17 @@
+<?php
+
+namespace Cognesy\Instructor\Extras\Evals\Data;
+
+use Cognesy\Instructor\Features\Schema\Attributes\Description;
+
+#[Description("The result of correctness evaluation.")]
+class BooleanCorrectnessAnalysis
+{
+    #[Description("Step by step assessment of the expected versus actual results.")]
+    public string $assessment;
+    #[Description("Decision if the actual result is correct.")]
+    public bool $isCorrect;
+    #[Description("If the result is incorrect - list of individual issues found in the actual result considering the expected values. Otherwise empty.")]
+    /** @var ParameterFeedback[] */
+    public array $feedback;
+}