Unified use of cached context across Instructor and Inference classes

cognesy · Oct 15, 2024 · 5a948ca · 5a948ca
1 parent 60e6989
commit 5a948ca
Show file tree

Hide file tree

Showing 67 changed files with 1,408 additions and 625 deletions.
diff --git a/90, b/90,
diff --git a/92, b/92,
diff --git a/composer.json b/composer.json
@@ -30,59 +30,59 @@
         }
     ],
     "require-dev": {
-        "pestphp/pest": "^2.34",
-        "symfony/var-dumper": "^6.4 || ^7.0",
-        "mockery/mockery": "^1.6",
-        "toolkit/cli-utils": "^2.0",
+        "ext-curl": "*",
+        "ext-dom": "*",
         "cebe/markdown": "^1.2",
-        "phpstan/phpstan": "^1.11",
-        "spatie/php-structure-discoverer": "^2.1",
-        "vimeo/psalm": "dev-master",
+        "duzun/hquery": "^3.1",
+        "gioni06/gpt3-tokenizer": "^1.2",
+        "guzzlehttp/psr7": "^2.7",
         "illuminate/database": "^11.10",
         "illuminate/events": "^11.10",
-        "spatie/yaml-front-matter": "^2.0",
-        "ext-curl": "*",
         "league/html-to-markdown": "^5.1",
+        "mockery/mockery": "^1.6",
+        "nyholm/psr7": "^1.8",
+        "pestphp/pest": "^2.34",
+        "php-http/discovery": "^1.20",
+        "phpstan/phpstan": "^1.11",
+        "psr/http-factory-implementation": "*",
+        "psy/psysh": "@stable",
+        "spatie/array-to-xml": "^3.3",
         "spatie/browsershot": "^4.1",
+        "spatie/php-structure-discoverer": "^2.1",
+        "spatie/yaml-front-matter": "^2.0",
         "symfony/browser-kit": "^7.1",
         "symfony/css-selector": "^7.1",
         "symfony/dom-crawler": "^7.1",
-        "duzun/hquery": "^3.1",
-        "ext-dom": "*",
-        "psy/psysh": "@stable",
         "symfony/http-client": "^7.1",
-        "php-http/discovery": "^1.20",
-        "psr/http-factory-implementation": "*",
         "symfony/psr-http-message-bridge": "^7.1",
-        "nyholm/psr7": "^1.8",
-        "guzzlehttp/psr7": "^2.7",
-        "twig/twig": "^3.0"
+        "symfony/var-dumper": "^6.4 || ^7.0",
+        "toolkit/cli-utils": "^2.0",
+        "twig/twig": "^3.0",
+        "vimeo/psalm": "dev-master"
     },
     "config": {
         "allow-plugins": {
             "pestphp/pest-plugin": true,
             "php-http/discovery": true
-        }
+        },
+        "sort-packages": true
     },
     "require": {
         "php": "^8.2",
         "ext-fileinfo": "*",
-        "gioni06/gpt3-tokenizer": "^1.2",
+        "adbario/php-dot-notation": "^3.3",
         "guzzlehttp/guzzle": "^7.8",
         "phpdocumentor/reflection-docblock": "^5.4",
         "phpstan/phpdoc-parser": "^1.29",
-        "psr/log": "^3.0",
         "psr/event-dispatcher": "^1.0",
-        "ramsey/uuid": "^4.7",
+        "psr/log": "^3.0",
+        "symfony/intl": "^7.1",
         "symfony/property-access": "^6.4 || ^7.0",
         "symfony/property-info": "^6.4 || ^7.0",
         "symfony/serializer": "^6.4 || ^7.0",
         "symfony/type-info": "^7.1",
         "symfony/validator": "^6.4 || ^7.0",
-        "vlucas/phpdotenv": "^5.6",
-        "spatie/array-to-xml": "^3.3",
-        "adbario/php-dot-notation": "^3.3",
-        "symfony/intl": "^7.1"
+        "vlucas/phpdotenv": "^5.6"
     },
     "scripts": {
         "tests": "@php vendor/bin/pest",

diff --git a/docs/cookbook/examples/advanced/context_cache.mdx b/docs/cookbook/examples/advanced/context_cache.mdx
@@ -63,7 +63,7 @@ multiple requests.
 <?php
 $content = file_get_contents(__DIR__ . '/../../../README.md');
 
-$cached = (new Instructor)->withConnection('openai')->cacheContext(
+$cached = (new Instructor)->withConnection('openai')->withCachedContext(
     system: 'Your goal is to respond questions about the project described in the README.md file'
         . "\n\n# README.md\n\n" . $content,
     prompt: 'Respond to the user with a description of the project with JSON using schema:\n<|json_schema|>',

diff --git a/evals/ComplexExtraction/run.php b/evals/ComplexExtraction/run.php
@@ -1,15 +1,14 @@
 <?php
 
 use Cognesy\Instructor\Enums\Mode;
-use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExperiment;
-use Cognesy\Instructor\Extras\Evals\Contracts\Metric;
+use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExecution;
 use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
 use Cognesy\Instructor\Extras\Evals\Data\Feedback;
-use Cognesy\Instructor\Extras\Evals\Data\InferenceCases;
-use Cognesy\Instructor\Extras\Evals\Data\InstructorData;
+use Cognesy\Instructor\Extras\Evals\Execution;
 use Cognesy\Instructor\Extras\Evals\Experiment;
-use Cognesy\Instructor\Extras\Evals\Inference\RunInstructor;
-use Cognesy\Instructor\Extras\Evals\ExperimentSuite;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceCases;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InstructorData;
+use Cognesy\Instructor\Extras\Evals\Executors\RunInstructor;
 use Cognesy\Instructor\Extras\Evals\Metrics\PercentageCorrectness;
 use Cognesy\Instructor\Extras\Sequence\Sequence;
 use Cognesy\Instructor\Features\LLM\Data\Usage;
@@ -22,18 +21,18 @@ class Company {
     public int $foundingYear;
 }
 
-class CompanyEval implements CanEvaluateExperiment
+class CompanyEval implements CanEvaluateExecution
 {
     public array $expectations;
 
     public function __construct(array $expectations) {
         $this->expectations = $expectations;
     }
 
-    public function evaluate(Experiment $experiment) : Evaluation {
+    public function evaluate(Execution $execution) : Evaluation {
         $expectedEvents = $this->expectations['events'];
         /** @var Sequence $events */
-        $events = $experiment->response->value();
+        $events = $execution->response->value();
         $result = ($expectedEvents - count($events->list)) / $expectedEvents;
         return new Evaluation(
             metric: new PercentageCorrectness('found', $result),
@@ -57,7 +56,7 @@ public function evaluate(Experiment $experiment) : Evaluation {
     responseModel: Company::class,
 );
 
-$runner = new ExperimentSuite(
+$experiment = new Experiment(
     cases: InferenceCases::only(
         connections: ['openai', 'anthropic', 'gemini', 'cohere'],
         modes: [Mode::Tools],
@@ -67,4 +66,4 @@ public function evaluate(Experiment $experiment) : Evaluation {
     evaluators: new CompanyEval(expectations: ['events' => 12]),
 );
 
-$outputs = $runner->execute();
+$outputs = $experiment->execute();
diff --git a/evals/LLMModes/CompanyEval.php b/evals/LLMModes/CompanyEval.php
@@ -3,27 +3,27 @@
 namespace Cognesy\Evals\LLMModes;
 
 use Cognesy\Instructor\Enums\Mode;
-use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExperiment;
+use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExecution;
 use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
 use Cognesy\Instructor\Extras\Evals\Data\Feedback;
-use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Execution;
 use Cognesy\Instructor\Extras\Evals\Metrics\BooleanCorrectness;
 use Cognesy\Instructor\Features\LLM\Data\Usage;
 use Cognesy\Instructor\Utils\Str;
 
-class CompanyEval implements CanEvaluateExperiment
+class CompanyEval implements CanEvaluateExecution
 {
     private array $expectations;
 
     public function __construct(array $expectations) {
         $this->expectations = $expectations;
     }
 
-    public function evaluate(Experiment $experiment) : Evaluation {
-        $isCorrect = match ($experiment->mode) {
-            Mode::Text => $this->validateText($experiment),
-            Mode::Tools => $this->validateToolsData($experiment),
-            default => $this->validateDefault($experiment),
+    public function evaluate(Execution $execution) : Evaluation {
+        $isCorrect = match ($execution->mode) {
+            Mode::Text => $this->validateText($execution),
+            Mode::Tools => $this->validateToolsData($execution),
+            default => $this->validateDefault($execution),
         };
         return new Evaluation(
             metric: new BooleanCorrectness('is_correct', $isCorrect),
@@ -32,25 +32,27 @@ public function evaluate(Experiment $experiment) : Evaluation {
         );
     }
 
-    private function validateToolsData(Experiment $experiment) : bool {
-        $data = $experiment->response->toolsData;
-        return 'store_company' === ($data[0]['name'] ?? '')
-            && 'ACME' === ($data[0]['arguments']['name'] ?? '')
-            && 2020 === (int) ($data[0]['arguments']['year'] ?? 0);
+    // INTERNAL /////////////////////////////////////////////////
+
+    private function validateToolsData(Execution $execution) : bool {
+        $data = $execution->response->toolsData[0];
+        return 'store_company' === ($data['name'] ?? '')
+            && 'ACME' === ($data['arguments']['name'] ?? '')
+            && 2020 === (int) ($data['arguments']['year'] ?? 0);
     }
 
-    private function validateDefault(Experiment $experiment) : bool {
-        $decoded = json_decode($experiment->response->json(), true);
+    private function validateDefault(Execution $execution) : bool {
+        $decoded = $execution->response->json()->toArray();
         return $this->expectations['name'] === ($decoded['name'] ?? '')
-            && $this->expectations['foundingYear'] === ($decoded['year'] ?? 0);
+            && $this->expectations['year'] === ($decoded['year'] ?? 0);
     }
 
-    private function validateText(Experiment $experiment) : bool {
+    private function validateText(Execution $execution) : bool {
         return Str::contains(
-            $experiment->response->content(),
+            $execution->response->content(),
             [
                 $this->expectations['name'],
-                (string) $this->expectations['foundingYear']
+                (string) $this->expectations['year']
             ]
         );
     }

diff --git a/evals/LLMModes/run.php b/evals/LLMModes/run.php
@@ -3,14 +3,12 @@
 $loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
 $loader->add('Cognesy\\Evals\\', __DIR__ . '../../evals/');
 
-use Cognesy\Instructor\Extras\Evals\Aggregators\FirstMetric;
-use Cognesy\Instructor\Extras\Evals\Aggregators\SelectedMetric;
-use Cognesy\Instructor\Extras\Evals\Data\InferenceCases;
-use Cognesy\Instructor\Extras\Evals\Data\InferenceData;
-use Cognesy\Instructor\Extras\Evals\Data\InferenceSchema;
-use Cognesy\Instructor\Extras\Evals\Inference\RunInference;
-use Cognesy\Instructor\Extras\Evals\ExperimentSuite;
 use Cognesy\Evals\LLMModes\CompanyEval;
+use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceCases;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceData;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceSchema;
+use Cognesy\Instructor\Extras\Evals\Executors\RunInference;
 
 $data = new InferenceData(
     messages: [
@@ -42,7 +40,7 @@
     ),
 );
 
-$experiments = new ExperimentSuite(
+$experiment = new Experiment(
     cases: InferenceCases::except(
         connections: [],
         modes: [],
@@ -51,9 +49,8 @@
     executor: new RunInference($data),
     evaluators: new CompanyEval(expectations: [
         'name' => 'ACME',
-        'foundingYear' => 2020
+        'year' => 2020
     ]),
-    aggregator: new FirstMetric()
 );
 
-$outputs = $experiments->execute();
+$outputs = $experiment->execute();
diff --git a/evals/SimpleExtraction/Company.php b/evals/SimpleExtraction/Company.php
@@ -2,7 +2,11 @@
 
 namespace Cognesy\Evals\SimpleExtraction;
 
+use Cognesy\Instructor\Features\Schema\Attributes\Description;
+
 class Company {
+    #[Description("The name of the company")]
     public string $name;
-    public int $foundingYear;
+    #[Description("The year the company was founded")]
+    public int $year;
 }
diff --git a/evals/SimpleExtraction/CompanyEval.php b/evals/SimpleExtraction/CompanyEval.php
@@ -2,25 +2,25 @@
 
 namespace Cognesy\Evals\SimpleExtraction;
 
-use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExperiment;
+use Cognesy\Instructor\Extras\Evals\Contracts\CanEvaluateExecution;
 use Cognesy\Instructor\Extras\Evals\Data\Evaluation;
 use Cognesy\Instructor\Extras\Evals\Data\Feedback;
-use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Execution;
 use Cognesy\Instructor\Extras\Evals\Metrics\BooleanCorrectness;
 use Cognesy\Instructor\Features\LLM\Data\Usage;
 
-class CompanyEval implements CanEvaluateExperiment
+class CompanyEval implements CanEvaluateExecution
 {
     private array $expectations;
 
     public function __construct(array $expectations) {
         $this->expectations = $expectations;
     }
 
-    public function evaluate(Experiment $experiment) : Evaluation {
-        $company = $experiment->response->value();
+    public function evaluate(Execution $execution) : Evaluation {
+        $company = $execution->response->value();
         $isCorrect = $company->name === $this->expectations['name']
-            && $company->foundingYear === $this->expectations['foundingYear'];
+            && $company->year === $this->expectations['year'];
         return new Evaluation(
             metric: new BooleanCorrectness('is_correct', $isCorrect),
             feedback: Feedback::none(),

diff --git a/evals/SimpleExtraction/run.php b/evals/SimpleExtraction/run.php
@@ -1,13 +1,12 @@
 <?php
 
-use Cognesy\Instructor\Enums\Mode;
-use Cognesy\Instructor\Extras\Evals\Aggregators\FirstMetric;
-use Cognesy\Instructor\Extras\Evals\Data\InferenceCases;
-use Cognesy\Instructor\Extras\Evals\Data\InstructorData;
-use Cognesy\Instructor\Extras\Evals\Inference\RunInstructor;
-use Cognesy\Instructor\Extras\Evals\ExperimentSuite;
-use Cognesy\Evals\SimpleExtraction\CompanyEval;
 use Cognesy\Evals\SimpleExtraction\Company;
+use Cognesy\Evals\SimpleExtraction\CompanyEval;
+use Cognesy\Instructor\Enums\Mode;
+use Cognesy\Instructor\Extras\Evals\Experiment;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InferenceCases;
+use Cognesy\Instructor\Extras\Evals\Executors\Data\InstructorData;
+use Cognesy\Instructor\Extras\Evals\Executors\RunInstructor;
 
 $loader = require 'vendor/autoload.php';
 $loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/');
@@ -21,7 +20,7 @@
     responseModel: Company::class,
 );
 
-$runner = new ExperimentSuite(
+$experiment = new Experiment(
     cases: InferenceCases::except(
         connections: ['ollama'],
         modes: [Mode::Text],
@@ -30,9 +29,8 @@
     executor: new RunInstructor($data),
     evaluators: new CompanyEval(expectations: [
         'name' => 'ACME',
-        'foundingYear' => 2020
+        'year' => 2020
     ]),
-    aggregator: new FirstMetric()
 );
 
-$outputs = $runner->execute();
+$outputs = $experiment->execute();
diff --git a/examples/A02_Advanced/ContextCaching/run.php b/examples/A02_Advanced/ContextCaching/run.php
@@ -63,7 +63,7 @@ class Project {
 <?php
 $content = file_get_contents(__DIR__ . '/../../../README.md');
 
-$cached = (new Instructor)->withConnection('openai')->cacheContext(
+$cached = (new Instructor)->withConnection('openai')->withCachedContext(
     system: 'Your goal is to respond questions about the project described in the README.md file'
         . "\n\n# README.md\n\n" . $content,
     prompt: 'Respond to the user with a description of the project with JSON using schema:\n<|json_schema|>',