From 066a205c175aab70aea4d4487b30d6712e60cb0d Mon Sep 17 00:00:00 2001 From: ddebowczyk Date: Thu, 10 Oct 2024 09:25:17 +0200 Subject: [PATCH] Evals + bug fixes --- config/debug.php | 1 + evals/LLMModes/run.php | 40 ++++--- evals/SimpleExtraction/run.php | 69 +++++------ src/Enums/Mode.php | 2 +- src/Extras/Evals/Combination.php | 99 ++++++++++++++++ src/Extras/Evals/Console/Display.php | 13 ++- src/Extras/Evals/Contracts/CanEvaluate.php | 11 ++ .../Evals/Contracts/CanExecuteExperiment.php | 2 +- src/Extras/Evals/Contracts/CanMapValues.php | 19 +++ src/Extras/Evals/Contracts/Metric.php | 10 ++ src/Extras/Evals/Data/EvalInput.php | 20 ++-- src/Extras/Evals/Data/EvalOutput.php | 3 +- src/Extras/Evals/Data/EvalSchema.php | 8 +- src/Extras/Evals/Evaluator.php | 83 +++++++------ .../Evals/Inference/InferenceAdapter.php | 99 ++++++++++++++++ src/Extras/Evals/Inference/InferenceModes.php | 109 ------------------ src/Extras/Evals/Inference/RunInference.php | 58 ++++------ src/Extras/Evals/Instructor/RunInstructor.php | 60 +++++----- src/Extras/Evals/Mappings/ConnectionModes.php | 20 ++++ src/Extras/Evals/Metrics/BooleanMetric.php | 31 +++++ .../Scalar/Traits/ProvidesJsonSchema.php | 2 +- src/Features/Http/Drivers/GuzzleDriver.php | 1 + src/Features/Http/Drivers/SymfonyDriver.php | 2 + src/Features/LLM/Drivers/CohereV2Driver.php | 4 +- src/Utils/Cli/Console.php | 14 ++- src/Utils/Debug/Debug.php | 10 ++ 26 files changed, 493 insertions(+), 297 deletions(-) create mode 100644 src/Extras/Evals/Combination.php create mode 100644 src/Extras/Evals/Contracts/CanEvaluate.php create mode 100644 src/Extras/Evals/Contracts/CanMapValues.php create mode 100644 src/Extras/Evals/Contracts/Metric.php create mode 100644 src/Extras/Evals/Inference/InferenceAdapter.php delete mode 100644 src/Extras/Evals/Inference/InferenceModes.php create mode 100644 src/Extras/Evals/Mappings/ConnectionModes.php create mode 100644 src/Extras/Evals/Metrics/BooleanMetric.php diff --git a/config/debug.php b/config/debug.php index cf708e19..174bda91 100644 --- a/config/debug.php +++ b/config/debug.php @@ -3,6 +3,7 @@ 'http' => [ 'enabled' => false, 'trace' => false, + 'requestUrl' => true, 'requestHeaders' => true, 'requestBody' => true, 'responseHeaders' => true, diff --git a/evals/LLMModes/run.php b/evals/LLMModes/run.php index 3c8dde6d..6b0df96e 100644 --- a/evals/LLMModes/run.php +++ b/evals/LLMModes/run.php @@ -4,24 +4,26 @@ $loader->add('Cognesy\\Evals\\', __DIR__ . '../../evals/'); use Cognesy\Instructor\Enums\Mode; +use Cognesy\Instructor\Extras\Evals\Combination; use Cognesy\Instructor\Extras\Evals\Data\EvalInput; use Cognesy\Instructor\Extras\Evals\Data\EvalSchema; use Cognesy\Instructor\Extras\Evals\Evaluator; use Cognesy\Instructor\Extras\Evals\Inference\RunInference; +use Cognesy\Instructor\Extras\Evals\Mappings\ConnectionModes; use Cognesy\Instructor\Utils\Str; $connections = [ -// 'azure', -// 'cohere1', -// 'cohere2', -// 'fireworks', -// 'gemini', + 'azure', + 'cohere1', + 'cohere2', + 'fireworks', + 'gemini', 'groq', -// 'mistral', -// 'ollama', -// 'openai', -// 'openrouter', -// 'together', + 'mistral', + 'ollama', + 'openai', + 'openrouter', + 'together', ]; $streamingModes = [ @@ -45,6 +47,15 @@ // azure, Mode::JsonSchema, sync|stream // +$combinations = Combination::generator( + mapping: ConnectionModes::class, + sources: [ + 'isStreaming' => $streamingModes, + 'mode' => $modes, + 'connection' => $connections, + ], +); + function evalFn(EvalInput $er) { $decoded = json_decode($er->response->json(), true); $isCorrect = match($er->mode) { @@ -93,12 +104,13 @@ function validateToolsData(array $data) : bool { ['role' => 'user', 'content' => 'What is the name and founding year of our company?'], ], schema: $schema, - executorClass: RunInference::class, + runner: new RunInference(), evalFn: fn(EvalInput $evalInput) => evalFn($evalInput), ); $outputs = $evaluator->execute( - connections: $connections, - modes: $modes, - streamingModes: $streamingModes +// connections: $connections, +// modes: $modes, +// streamingModes: $streamingModes + combinations: $combinations ); diff --git a/evals/SimpleExtraction/run.php b/evals/SimpleExtraction/run.php index d3e3f64b..fef4be7b 100644 --- a/evals/SimpleExtraction/run.php +++ b/evals/SimpleExtraction/run.php @@ -1,25 +1,28 @@ add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); $connections = [ - 'azure', - 'cohere1', +// 'azure', +// 'cohere1', 'cohere2', - 'fireworks', - 'gemini', - 'groq', - 'mistral', - 'ollama', - 'openai', - 'openrouter', - 'together', +// 'fireworks', +// 'gemini', +// 'groq', +// 'mistral', +// 'ollama', +// 'openai', +// 'openrouter', +// 'together', ]; $streamingModes = [ @@ -34,18 +37,20 @@ Mode::Tools, ]; -//$report = file_get_contents(__DIR__ . '/report.txt'); -//$examples = require 'examples.php'; -//$prompt = 'Extract a list of project events with all the details from the provided input in JSON format using schema: <|json_schema|>'; -//$responseModel = Sequence::of(ProjectEvent::class); +$combinations = Combination::generator( + mapping: ConnectionModes::class, + sources: [ + 'isStreaming' => $streamingModes, + 'mode' => $modes, + 'connection' => $connections, + ], +); class Company { public string $name; public int $foundingYear; } -//Debug::enable(); - function evalFn(EvalInput $er) { /** @var Person $decoded */ $person = $er->response->value(); @@ -53,6 +58,13 @@ function evalFn(EvalInput $er) { && $person->foundingYear === 2020; } +//Debug::enable(); + +//$report = file_get_contents(__DIR__ . '/report.txt'); +//$examples = require 'examples.php'; +//$prompt = 'Extract a list of project events with all the details from the provided input in JSON format using schema: <|json_schema|>'; +//$responseModel = Sequence::of(ProjectEvent::class); + $outputs = (new Evaluator( messages: [ ['role' => 'user', 'content' => 'YOUR GOAL: Use tools to store the information from context based on user questions.'], @@ -62,29 +74,8 @@ function evalFn(EvalInput $er) { ['role' => 'user', 'content' => 'What is the name and founding year of our company?'], ], schema: Company::class, - executorClass: RunInstructor::class, + runner: new RunInstructor(), evalFn: fn(EvalInput $er) => evalFn($er), ))->execute( - connections: $connections, - modes: $modes, - streamingModes: $streamingModes + combinations: $combinations ); - - -//$connection = 'gemini'; -//$mode = Mode::Json; -//$withStreaming = false; -// -//$action = new ExtractData( -// messages: $input, -// responseModel: $responseModel, -// connection: $connection, -// mode: $mode, -// withStreaming: $withStreaming, -// prompt: $prompt, -// examples: $examples, -// model: 'gemini-1.5-pro', -//); -// -//$response = $action(); -//dump($response->get()->all()); diff --git a/src/Enums/Mode.php b/src/Enums/Mode.php index 63e078f6..12b4f859 100644 --- a/src/Enums/Mode.php +++ b/src/Enums/Mode.php @@ -7,7 +7,7 @@ enum Mode : string case Tools = 'tool_call'; case Json = 'json'; case JsonSchema = 'json_schema'; - case MdJson = 'markdown_json'; + case MdJson = 'md_json'; case Text = 'text'; // unstructured text response public function is(array|Mode $mode) : bool { diff --git a/src/Extras/Evals/Combination.php b/src/Extras/Evals/Combination.php new file mode 100644 index 00000000..4a5a4175 --- /dev/null +++ b/src/Extras/Evals/Combination.php @@ -0,0 +1,99 @@ + $mapping The fully qualified class name that implements CanMapValues. + * @param array $sources Associative array mapping keys to their respective iterables. + * + * @return Generator Yields instances of the mapping class for each combination. + * + * @throws InvalidArgumentException If any key in order is missing from sources. + */ + public static function generator( + string $mapping, + array $sources + ): Generator { + $order = array_keys($sources); + // Ensure all keys in order exist in sources + foreach ($order as $key) { + if (!array_key_exists($key, $sources)) { + throw new InvalidArgumentException("Source for key '{$key}' not provided."); + } + } + + // Initialize iterators for each key in the specified order + $iterators = []; + foreach ($order as $key) { + $iterators[$key] = self::getIterator($sources[$key]); + } + + // Start the recursive generation of combinations + yield from self::generateCombinations($mapping, $order, $iterators, []); + } + + /** + * Recursively generates combinations of values. + * + * @template T + * + * @param class-string $mapping + * @param string[] $keys + * @param array $iterators + * @param array $currentCombination + * + * @return Generator + */ + private static function generateCombinations( + string $mapping, + array $keys, + array $iterators, + array $currentCombination + ): Generator { + $key = array_shift($keys); + + if ($key === null) { + // Base case: all keys have been processed + yield $mapping::map($currentCombination); + return; + } + + foreach ($iterators[$key] as $value) { + $newCombination = $currentCombination; + $newCombination[$key] = $value; + + // Recursively generate combinations for the remaining keys + yield from self::generateCombinations($mapping, $keys, $iterators, $newCombination); + } + } + + /** + * Converts an iterable into an Iterator. + * + * @param iterable $iterable + * @return Iterator + */ + private static function getIterator(iterable $iterable): Iterator + { + if ($iterable instanceof Iterator) { + return $iterable; + } + + return new ArrayIterator(array: $iterable); + } +} diff --git a/src/Extras/Evals/Console/Display.php b/src/Extras/Evals/Console/Display.php index c5b2c73e..e665a1db 100644 --- a/src/Extras/Evals/Console/Display.php +++ b/src/Extras/Evals/Console/Display.php @@ -13,9 +13,9 @@ class Display { public function before(Mode $mode, string $connection, bool $isStreamed) : void { echo Console::columns([ - [14, $mode->value, STR_PAD_RIGHT, Color::YELLOW], - [12, $connection, STR_PAD_RIGHT, Color::WHITE], - [10, $isStreamed ? 'stream' : 'sync', STR_PAD_LEFT, $isStreamed ? Color::BLUE : Color::DARK_BLUE], + [10, $connection, STR_PAD_RIGHT, Color::WHITE], + [11, $mode->value, STR_PAD_RIGHT, Color::YELLOW], + [8, $isStreamed ? 'stream' : 'sync', STR_PAD_LEFT, $isStreamed ? Color::BLUE : Color::DARK_BLUE], ], 80); Console::print('', [Color::GRAY, Color::BG_BLACK]); } @@ -23,7 +23,7 @@ public function before(Mode $mode, string $connection, bool $isStreamed) : void public function after(EvalOutput $evalResponse) : void { $answer = $evalResponse->notes; $answerLine = str_replace("\n", '\n', $answer); - $isCorrect = $evalResponse->isCorrect; + $metric = $evalResponse->metric; $timeElapsed = $evalResponse->timeElapsed; $tokensPerSec = $evalResponse->outputTps(); $exception = $evalResponse->exception; @@ -34,14 +34,15 @@ public function after(EvalOutput $evalResponse) : void { //Console::println(, [Color::RED, Color::BG_BLACK]); echo Console::columns([ [9, '', STR_PAD_LEFT, [Color::DARK_YELLOW]], - [5, ' !!!!', STR_PAD_RIGHT, [Color::WHITE, COLOR::BOLD, Color::BG_MAGENTA]], + [10, '', STR_PAD_LEFT, [Color::CYAN]], + [6, '!!!', STR_PAD_BOTH, [Color::WHITE, COLOR::BOLD, Color::BG_MAGENTA]], [60, ' ' . $this->exc2txt($exception, 80), STR_PAD_RIGHT, [Color::RED, Color::BG_BLACK]], ], 120); } else { echo Console::columns([ [9, $this->timeFormat($timeElapsed), STR_PAD_LEFT, [Color::DARK_YELLOW]], [10, $this->tokensPerSecFormat($tokensPerSec), STR_PAD_LEFT, [Color::CYAN]], - [5, $isCorrect ? ' OK ' : ' FAIL', STR_PAD_RIGHT, $isCorrect ? [Color::BG_GREEN, Color::WHITE] : [Color::BG_RED, Color::WHITE]], + [6, $metric->toString(), STR_PAD_BOTH, $metric->toCliColor()], [60, ' ' . $answerLine, STR_PAD_RIGHT, [Color::WHITE, Color::BG_BLACK]], ], 120); } diff --git a/src/Extras/Evals/Contracts/CanEvaluate.php b/src/Extras/Evals/Contracts/CanEvaluate.php new file mode 100644 index 00000000..cd1325f1 --- /dev/null +++ b/src/Extras/Evals/Contracts/CanEvaluate.php @@ -0,0 +1,11 @@ + $values + * @return T + */ + public static function map(array $values); +} diff --git a/src/Extras/Evals/Contracts/Metric.php b/src/Extras/Evals/Contracts/Metric.php new file mode 100644 index 00000000..e2b5ee52 --- /dev/null +++ b/src/Extras/Evals/Contracts/Metric.php @@ -0,0 +1,10 @@ +schema; + } + public function evalSchema() : EvalSchema { + // TODO: make it accept any and use SchemaFactory to generate EvalSchema object if (!$this->schema instanceof EvalSchema) { throw new \Exception('Schema is not an instance of EvalSchema.'); } diff --git a/src/Extras/Evals/Data/EvalOutput.php b/src/Extras/Evals/Data/EvalOutput.php index a7866627..86214b85 100644 --- a/src/Extras/Evals/Data/EvalOutput.php +++ b/src/Extras/Evals/Data/EvalOutput.php @@ -2,6 +2,7 @@ namespace Cognesy\Instructor\Extras\Evals\Data; +use Cognesy\Instructor\Extras\Evals\Contracts\Metric; use Exception; class EvalOutput @@ -9,7 +10,7 @@ class EvalOutput public function __construct( public string $id = '', public string $notes = '', - public bool $isCorrect = false, + public ?Metric $metric = null, public float $timeElapsed = 0.0, public ?Exception $exception = null, public int $inputTokens = 0, diff --git a/src/Extras/Evals/Data/EvalSchema.php b/src/Extras/Evals/Data/EvalSchema.php index e5ce6d3d..fe2aec0c 100644 --- a/src/Extras/Evals/Data/EvalSchema.php +++ b/src/Extras/Evals/Data/EvalSchema.php @@ -2,7 +2,9 @@ namespace Cognesy\Instructor\Extras\Evals\Data; -class EvalSchema +use Cognesy\Instructor\Contracts\CanProvideJsonSchema; + +class EvalSchema implements CanProvideJsonSchema { public function __construct( private string $toolName, @@ -52,4 +54,8 @@ public function toolChoice() : array { ] ]; } + + public function toJsonSchema(): array { + return $this->schema; + } } diff --git a/src/Extras/Evals/Evaluator.php b/src/Extras/Evals/Evaluator.php index 8519db14..80327c97 100644 --- a/src/Extras/Evals/Evaluator.php +++ b/src/Extras/Evals/Evaluator.php @@ -7,48 +7,50 @@ use Cognesy\Instructor\Extras\Evals\Contracts\CanExecuteExperiment; use Cognesy\Instructor\Extras\Evals\Data\EvalInput; use Cognesy\Instructor\Extras\Evals\Data\EvalOutput; +use Cognesy\Instructor\Extras\Evals\Mappings\ConnectionModes; +use Cognesy\Instructor\Extras\Evals\Metrics\BooleanMetric; use Exception; +use Generator; class Evaluator { private array $exceptions = []; private array $responses = []; private Display $display; private string|array|object $schema; - /** @var class-string */ - private string $executor; + private CanExecuteExperiment $runner; private string|array $messages; private Closure $evalFn; public function __construct( - string|array $messages, - string|array|object $schema, - string $executorClass, - Closure $evalFn, + string|array $messages, + string|array|object $schema, + CanExecuteExperiment $runner, + Closure $evalFn, ) { $this->messages = $messages; $this->schema = $schema; - $this->executor = $executorClass; + $this->runner = $runner; $this->evalFn = $evalFn; $this->display = new Display(); } // PUBLIC ////////////////////////////////////////////////// + /** + * @param Generator $combinations + * @return array + */ public function execute( - array $connections, - array $modes, - array $streamingModes + Generator $combinations ) : array { - foreach ($streamingModes as $isStreamed) { - foreach ($modes as $mode) { - foreach ($connections as $connection) { - $this->display->before($mode, $connection, $isStreamed); - $evalResponse = $this->executeSingle($connection, $mode, $isStreamed); - $this->responses[] = $evalResponse; - $this->display->after($evalResponse); - } - } + foreach ($combinations as $params) { + $this->display->before($params->mode, $params->connection, $params->isStreaming); + $evalInput = $this->makeEvalInput($params->connection, $params->mode, $params->isStreaming); + $evalResponse = $this->executeSingle($evalInput); + $this->responses[] = $evalResponse; + $this->display->after($evalResponse); } + if (!empty($this->exceptions)) { $this->display->displayExceptions($this->exceptions); } @@ -57,48 +59,34 @@ public function execute( // INTERNAL ///////////////////////////////////////////////// - private function executeSingle( - string $connection, - Mode $mode, - bool $isStreamed - ) : EvalOutput { - $key = $this->makeKey($connection, $mode, $isStreamed); + private function executeSingle(EvalInput $evalInput) : EvalOutput { try { - $evalInput = new EvalInput( - messages: $this->messages, - schema: $this->schema, - mode: $mode, - connection: $connection, - isStreamed: $isStreamed, - ); - // execute and measure time $time = microtime(true); - /** @var CanExecuteExperiment $experiment */ - $experiment = ($this->executor)::fromEvalInput($evalInput); - $experiment->execute(); - $llmResponse = $experiment->getLLMResponse(); + $this->runner->withEvalInput($evalInput); + $this->runner->execute(); + $llmResponse = $this->runner->getLLMResponse(); $evalInput->withResponse($llmResponse); $timeElapsed = microtime(true) - $time; $isCorrect = ($this->evalFn)($evalInput); $evalResponse = new EvalOutput( - id: $key, + id: $evalInput->id, notes: $llmResponse->content(), - isCorrect: $isCorrect, + metric: new BooleanMetric($isCorrect), timeElapsed: $timeElapsed, inputTokens: $llmResponse->usage()->inputTokens, outputTokens: $llmResponse->usage()->outputTokens, ); } catch(Exception $e) { $timeElapsed = microtime(true) - $time; - $this->exceptions[$key] = $e; + $this->exceptions[$evalInput->id] = $e; $evalResponse = new EvalOutput( - id: $key, + id: $evalInput->id, notes: '', - isCorrect: false, + metric: new BooleanMetric(false), timeElapsed: $timeElapsed, exception: $e, ); @@ -106,6 +94,17 @@ private function executeSingle( return $evalResponse; } + private function makeEvalInput(string $connection, Mode $mode, bool $isStreamed) : EvalInput { + return new EvalInput( + id: $this->makeKey($connection, $mode, $isStreamed), + messages: $this->messages, + schema: $this->schema, + mode: $mode, + connection: $connection, + isStreamed: $isStreamed, + ); + } + private function makeKey(string $connection, Mode $mode, bool $isStreamed) : string { return $connection.'::'.$mode->value.'::'.($isStreamed ? 'streamed' : 'sync'); } diff --git a/src/Extras/Evals/Inference/InferenceAdapter.php b/src/Extras/Evals/Inference/InferenceAdapter.php new file mode 100644 index 00000000..57c1f139 --- /dev/null +++ b/src/Extras/Evals/Inference/InferenceAdapter.php @@ -0,0 +1,99 @@ + 'user', 'content' => $messages]]; + $options = [ + 'max_tokens' => $maxTokens, + 'stream' => $isStreamed + ]; + $inferenceResponse = match($mode) { + Mode::Tools => $this->forModeTools($connection, $messages, $evalSchema, $options), + Mode::JsonSchema => $this->forModeJsonSchema($connection, $messages, $evalSchema, $options), + Mode::Json => $this->forModeJson($connection, $messages, $evalSchema, $options), + Mode::MdJson => $this->forModeMdJson($connection, $messages, $evalSchema, $options), + Mode::Text => $this->forModeText($connection, $messages, $options), + }; + return $inferenceResponse->response(); + } + + public function forModeTools(string $connection, string|array $messages, EvalSchema $schema, array $options) : InferenceResponse { + return (new Inference) + ->withConnection($connection) + ->create( + messages: $messages, + tools: $schema->tools(), + toolChoice: $schema->toolChoice(), + options: $options, + mode: Mode::Tools, + ); + } + + public function forModeJsonSchema(string $connection, string|array $messages, EvalSchema $schema, array $options) : InferenceResponse { + return (new Inference) + ->withConnection($connection) + ->create( + messages: array_merge($messages, [ + ['role' => 'user', 'content' => 'Use JSON Schema: ' . json_encode($schema->schema())], + ['role' => 'user', 'content' => 'Respond with correct JSON.'], + ]), + responseFormat: $schema->responseFormatJsonSchema(), + options: $options, + mode: Mode::JsonSchema, + ); + } + + public function forModeJson(string $connection, string|array $messages, EvalSchema $schema, array $options) : InferenceResponse { + return (new Inference) + ->withConnection($connection) + ->create( + messages: array_merge($messages, [ + ['role' => 'user', 'content' => 'Use JSON Schema: ' . json_encode($schema->schema())], + ['role' => 'user', 'content' => 'Respond with correct JSON.'], + ]), + responseFormat: $schema->responseFormatJson(), + options: $options, + mode: Mode::Json, + ); + } + + public function forModeMdJson(string $connection, string|array $messages, EvalSchema $schema, array $options) : InferenceResponse { + return (new Inference) + ->withConnection($connection) + ->create( + messages: array_merge($messages, [ + ['role' => 'user', 'content' => 'Use JSON Schema: ' . json_encode($schema->schema())], + ['role' => 'user', 'content' => 'Respond with correct JSON'], + ['role' => 'user', 'content' => '```json'], + ]), + options: $options, + mode: Mode::MdJson, + ); + } + + public function forModeText(string $connection, string|array $messages, array $options) : InferenceResponse { + return (new Inference) + ->withConnection($connection) + ->create( + messages: $messages, + options: $options, + mode: Mode::Text, + ); + } +} \ No newline at end of file diff --git a/src/Extras/Evals/Inference/InferenceModes.php b/src/Extras/Evals/Inference/InferenceModes.php deleted file mode 100644 index feddd49b..00000000 --- a/src/Extras/Evals/Inference/InferenceModes.php +++ /dev/null @@ -1,109 +0,0 @@ -schema = $schema; - $this->maxTokens = $maxTokens; - } - - public function schema() : array { - return $this->schema->schema(); - } - - public function callInferenceFor( - string|array $messages, - Mode $mode, - string $connection, - array $schema, - bool $isStreamed - ) : LLMResponse { - $messages = is_array($messages) ? $messages : [['role' => 'user', 'content' => $messages]]; - $inferenceResponse = match($mode) { - Mode::Tools => $this->forModeTools($messages, $connection, $schema, $isStreamed), - Mode::JsonSchema => $this->forModeJsonSchema($messages, $connection, $schema, $isStreamed), - Mode::Json => $this->forModeJson($messages, $connection, $schema, $isStreamed), - Mode::MdJson => $this->forModeMdJson($messages, $connection, $schema, $isStreamed), - Mode::Text => $this->forModeText($messages, $connection, $isStreamed), - }; - return $inferenceResponse->response(); - } - - public function forModeTools(string|array $query, string $connection, array $schema, bool $isStreamed) : InferenceResponse { - return (new Inference) - ->withConnection($connection) - ->create( - messages: $query, - tools: $this->schema->tools(), - toolChoice: $this->schema->toolChoice(), - options: ['max_tokens' => $this->maxTokens, 'stream' => $isStreamed], - mode: Mode::Tools, - ); - } - - public function forModeJsonSchema(string|array $query, string $connection, array $schema, bool $isStreamed) : InferenceResponse { - return (new Inference) - ->withConnection($connection) - ->create( - messages: array_merge($query, [ - ['role' => 'user', 'content' => 'Use JSON Schema: ' . json_encode($schema)], - ['role' => 'user', 'content' => 'Respond with correct JSON.'], - ]), - responseFormat: $this->schema->responseFormatJsonSchema(), - options: ['max_tokens' => $this->maxTokens, 'stream' => $isStreamed], - mode: Mode::JsonSchema, - ); - } - - public function forModeJson(string|array $query, string $connection, array $schema, bool $isStreamed) : InferenceResponse { - return (new Inference) - ->withConnection($connection) - ->create( - messages: array_merge($query, [ - ['role' => 'user', 'content' => 'Use JSON Schema: ' . json_encode($schema)], - ['role' => 'user', 'content' => 'Respond with correct JSON.'], - ]), - responseFormat: $this->schema->responseFormatJson(), - options: ['max_tokens' => $this->maxTokens, 'stream' => $isStreamed], - mode: Mode::Json, - ); - } - - public function forModeMdJson(string|array $query, string $connection, array $schema, bool $isStreamed) : InferenceResponse { - return (new Inference) - ->withConnection($connection) - ->create( - messages: array_merge($query, [ - ['role' => 'user', 'content' => 'Use JSON Schema: ' . json_encode($schema)], - ['role' => 'user', 'content' => 'Respond with correct JSON'], - ['role' => 'user', 'content' => '```json'], - ]), - options: ['max_tokens' => $this->maxTokens, 'stream' => $isStreamed], - mode: Mode::MdJson, - ); - } - - public function forModeText(string|array $query, string $connection, bool $isStreamed) : InferenceResponse { - return (new Inference) - ->withConnection($connection) - ->create( - messages: $query, - options: ['max_tokens' => $this->maxTokens, 'stream' => $isStreamed], - mode: Mode::Text, - ); - } -} \ No newline at end of file diff --git a/src/Extras/Evals/Inference/RunInference.php b/src/Extras/Evals/Inference/RunInference.php index 51d3e0fa..130f4a9d 100644 --- a/src/Extras/Evals/Inference/RunInference.php +++ b/src/Extras/Evals/Inference/RunInference.php @@ -10,44 +10,30 @@ class RunInference implements CanExecuteExperiment { - private InferenceModes $modes; - private string|array $query; - private LLMResponse $llmResponse; + private InferenceAdapter $inferenceAdapter; + + private string|array $messages; private Mode $mode; private string $connection; private bool $isStreamed; + private int $maxTokens; + private EvalSchema $schema; private string $answer; private LLMResponse $response; - public function __construct( - string|array $query, - EvalSchema $schema, - Mode $mode, - string $connection, - bool $isStreamed, - int $maxTokens, - ) { - $this->query = $query; - $this->modes = new InferenceModes( - schema: $schema, - maxTokens: $maxTokens - ); - $this->mode = $mode; - $this->connection = $connection; - $this->isStreamed = $isStreamed; + public function __construct() { + $this->inferenceAdapter = new InferenceAdapter(); } - public static function fromEvalInput(EvalInput $input) : self { - $instance = new RunInference( - query: $input->messages, - schema: $input->evalSchema(), - mode: $input->mode, - connection: $input->connection, - isStreamed: $input->isStreamed, - maxTokens: $input->maxTokens, - ); - return $instance; + public function withEvalInput(EvalInput $input) : self { + $this->messages = $input->messages; + $this->mode = $input->mode; + $this->connection = $input->connection; + $this->isStreamed = $input->isStreamed; + $this->maxTokens = $input->maxTokens; + $this->schema = $input->evalSchema(); + return $this; } public function execute() : void { @@ -66,13 +52,13 @@ public function getLLMResponse() : LLMResponse { // INTERNAL ///////////////////////////////////////////////// private function makeLLMResponse() : LLMResponse { - $this->llmResponse = $this->modes->callInferenceFor( - $this->query, - $this->mode, - $this->connection, - $this->modes->schema(), - $this->isStreamed + return $this->inferenceAdapter->callInferenceFor( + messages: $this->messages, + mode: $this->mode, + connection: $this->connection, + evalSchema: $this->schema, + isStreamed: $this->isStreamed, + maxTokens: $this->maxTokens, ); - return $this->llmResponse; } } \ No newline at end of file diff --git a/src/Extras/Evals/Instructor/RunInstructor.php b/src/Extras/Evals/Instructor/RunInstructor.php index 9a736876..82d63a01 100644 --- a/src/Extras/Evals/Instructor/RunInstructor.php +++ b/src/Extras/Evals/Instructor/RunInstructor.php @@ -11,42 +11,36 @@ class RunInstructor implements CanExecuteExperiment { - public $maxTokens = 4096; - public $toolName = ''; - public $toolDescription = ''; - public $retryPrompt = ''; - public int $maxRetries = 0; + public string|array $messages = ''; + public string|array|object $responseModel = []; + public string $connection = ''; + public Mode $mode = Mode::Json; + public bool $withStreaming = false; + public int $maxTokens = 4096; + public string $toolName = ''; + public string $toolDescription = ''; + public string $system = ''; + public string $prompt = ''; + public string|array|object $input = ''; + public array $examples = []; + public string $model = ''; + public string $retryPrompt = ''; + public int $maxRetries = 0; private InstructorResponse $instructorResponse; - private mixed $answer; private LLMResponse $llmResponse; + private mixed $answer; - public function __construct( - readonly public string|array $messages, - readonly public string|array|object $responseModel, - readonly public string $connection, - readonly public Mode $mode = Mode::Json, - readonly public bool $withStreaming = false, - readonly public string $prompt = '', - readonly public string|array|object $input = '', - readonly public array $examples = [], - readonly public string $model = '', - ) {} - - public static function fromEvalInput(EvalInput $input) : self { - $instance = new RunInstructor( - messages: $input->messages, - responseModel: $input->schema, - connection: $input->connection, - mode: $input->mode, - withStreaming: $input->isStreamed, - prompt: '', - input: '', - examples: [], - model: '', - ); - return $instance; + public function withEvalInput(EvalInput $input) : self { + $this->messages = $input->messages; + $this->responseModel = $input->responseSchema(); + $this->connection = $input->connection; + $this->mode = $input->mode; + $this->withStreaming = $input->isStreamed; + //$this->toolName = $input->responseSchema()->toolName; + //$this->toolDescription = $input->responseSchema()->toolDescription; + return $this; } public function execute() : void { @@ -66,7 +60,7 @@ public function getLLMResponse() : LLMResponse { // INTERNAL ///////////////////////////////////////////////// private function makeInstructorResponse() : InstructorResponse { - $response = (new Instructor) + return (new Instructor) ->withConnection($this->connection) ->request( messages: $this->messages, @@ -86,7 +80,5 @@ private function makeInstructorResponse() : InstructorResponse { retryPrompt: $this->retryPrompt, mode: $this->mode, ); - $this->instructorResponse = $response; - return $response; } } diff --git a/src/Extras/Evals/Mappings/ConnectionModes.php b/src/Extras/Evals/Mappings/ConnectionModes.php new file mode 100644 index 00000000..f6108f34 --- /dev/null +++ b/src/Extras/Evals/Mappings/ConnectionModes.php @@ -0,0 +1,20 @@ +mode = $values['mode'] ?? Mode::Text; + $instance->connection = $values['connection'] ?? 'openai'; + $instance->isStreaming = $values['isStreaming'] ?? false; + return $instance; + } +} diff --git a/src/Extras/Evals/Metrics/BooleanMetric.php b/src/Extras/Evals/Metrics/BooleanMetric.php new file mode 100644 index 00000000..a6f67873 --- /dev/null +++ b/src/Extras/Evals/Metrics/BooleanMetric.php @@ -0,0 +1,31 @@ +value; + } + + public function toLoss(): float { + return $this->value ? 0 : 1; + } + + public function toScore(): float { + return $this->value ? 1 : 0; + } + + public function toString(): string { + return $this->value ? 'OK' : 'FAIL'; + } + + public function toCliColor(): array { + return $this->value ? [Color::BG_GREEN, Color::WHITE] : [Color::BG_RED, Color::WHITE]; + } +} \ No newline at end of file diff --git a/src/Extras/Scalar/Traits/ProvidesJsonSchema.php b/src/Extras/Scalar/Traits/ProvidesJsonSchema.php index d5b54ec7..ba3477bf 100644 --- a/src/Extras/Scalar/Traits/ProvidesJsonSchema.php +++ b/src/Extras/Scalar/Traits/ProvidesJsonSchema.php @@ -13,7 +13,6 @@ trait ProvidesJsonSchema public function toJsonSchema() : array { $name = $this->name; $array = [ - 'x-php-class' => Scalar::class, 'type' => 'object', 'properties' => [ $name => [ @@ -22,6 +21,7 @@ public function toJsonSchema() : array { 'type' => $this->type->toJsonType(), ], ], + 'x-php-class' => Scalar::class, ]; if (!empty($this->options)) { /** @noinspection UnsupportedStringOffsetOperationsInspection */ diff --git a/src/Features/Http/Drivers/GuzzleDriver.php b/src/Features/Http/Drivers/GuzzleDriver.php index 3bf92398..b19e0ba8 100644 --- a/src/Features/Http/Drivers/GuzzleDriver.php +++ b/src/Features/Http/Drivers/GuzzleDriver.php @@ -50,6 +50,7 @@ public function handle( bool $streaming = false ) : CanAccessResponse { $this->events->dispatch(new RequestSentToLLM($url, $method, $headers, $body)); + Debug::tryDumpUrl($url); try { $response = $this->client->request($method, $url, [ 'headers' => $headers, diff --git a/src/Features/Http/Drivers/SymfonyDriver.php b/src/Features/Http/Drivers/SymfonyDriver.php index b594d7de..675030b2 100644 --- a/src/Features/Http/Drivers/SymfonyDriver.php +++ b/src/Features/Http/Drivers/SymfonyDriver.php @@ -10,6 +10,7 @@ use Cognesy\Instructor\Features\Http\Contracts\CanAccessResponse; use Cognesy\Instructor\Features\Http\Contracts\CanHandleHttp; use Cognesy\Instructor\Features\Http\Data\HttpClientConfig; +use Cognesy\Instructor\Utils\Debug\Debug; use Exception; use Symfony\Component\HttpClient\HttpClient; use Symfony\Contracts\HttpClient\HttpClientInterface; @@ -36,6 +37,7 @@ public function handle( ): CanAccessResponse { $this->events->dispatch(new RequestSentToLLM($url, $method, $headers, $body)); try { + Debug::tryDumpUrl($url); $response = $this->client->request( method: $method, url: $url, diff --git a/src/Features/LLM/Drivers/CohereV2Driver.php b/src/Features/LLM/Drivers/CohereV2Driver.php index 32591e7a..5ebad31e 100644 --- a/src/Features/LLM/Drivers/CohereV2Driver.php +++ b/src/Features/LLM/Drivers/CohereV2Driver.php @@ -160,10 +160,10 @@ private function normalizeContent(array|string $content) : string { private function makeUsage(array $data) : Usage { return new Usage( - inputTokens: $data['meta']['billed_units']['input_tokens'] + inputTokens: $data['usage']['billed_units']['input_tokens'] ?? $data['delta']['usage']['billed_units']['input_tokens'] ?? 0, - outputTokens: $data['meta']['billed_units']['output_tokens'] + outputTokens: $data['usage']['billed_units']['output_tokens'] ?? $data['delta']['usage']['billed_units']['output_tokens'] ?? 0, cacheWriteTokens: 0, diff --git a/src/Utils/Cli/Console.php b/src/Utils/Cli/Console.php index 4c61b15a..4bee8020 100644 --- a/src/Utils/Cli/Console.php +++ b/src/Utils/Cli/Console.php @@ -3,6 +3,8 @@ class Console { + const COLUMN_DIVIDER = ' '; + public static function print(string $message, string|array $color = ''): void { print(self::color($color, $message)); } @@ -16,6 +18,12 @@ public static function clearScreen(): void { //echo chr(27).chr(91).'H'.chr(27).chr(91).'J'; } + public static function center(string $message, int $width, string|array $color = ''): string { + $message = self::color($color, $message); + $message = str_pad($message, $width, ' ', STR_PAD_BOTH); + return $message; + } + public static function columns(array $columns, int $maxWidth): string { $maxWidth = max($maxWidth, 80); $message = ''; @@ -35,7 +43,7 @@ public static function columns(array $columns, int $maxWidth): string { align: $row[2]??STR_PAD_RIGHT ); } - $message .= ' '; + $message .= self::COLUMN_DIVIDER; } return $message; } @@ -49,8 +57,8 @@ static private function toColumn(int $chars, mixed $text, int $align, string|arr ? '…'.substr($short,1) : substr($short, 0, -1).'…'; } - $output = str_pad($short, $chars, ' ', $align); - $output = self::color($color, $output); + $output = self::color($color, str_pad($short, $chars, ' ', $align)); + $output .= Color::RESET; return $output; } diff --git a/src/Utils/Debug/Debug.php b/src/Utils/Debug/Debug.php index 7c29dd03..de531b7a 100644 --- a/src/Utils/Debug/Debug.php +++ b/src/Utils/Debug/Debug.php @@ -96,4 +96,14 @@ private static function printBody(string $body) : void { /** @noinspection ForgottenDebugOutputInspection */ dump(json_decode($body)); } + + public static function tryDumpUrl(string $url) : void { + if (Debug::isFlag('http.requestUrl')) { + Console::println(""); + Console::println("[REQUEST URL]", [Color::YELLOW]); + Console::println($url, [Color::GRAY]); + Console::println("[REQUEST /URL]", [Color::YELLOW]); + Console::println(""); + } + } } \ No newline at end of file