From 091dcf2851d1c5c46295fce55f8956e916cfb40e Mon Sep 17 00:00:00 2001 From: ddebowczyk Date: Mon, 26 Aug 2024 21:43:49 +0200 Subject: [PATCH] Example of token consumption counting using events --- README.md | 5 + .../cookbook/examples/api_support/caching.mdx | 70 -------------- .../examples/basics/complex_extraction.mdx | 2 +- .../basics/complex_extraction_claude.mdx | 1 - .../examples/techniques/image_to_data.mdx | 2 +- .../techniques/image_to_data_anthropic.mdx | 74 +++++++++++++++ .../troubleshooting/token_usage_events.mdx | 94 +++++++++++++++++++ docs/mint.json | 3 +- examples/01_Basics/ComplexExtraction/run.php | 2 +- examples/03_Techniques/ImageToData/run.php | 2 +- .../ImageToDataAnthropic/run.php | 4 +- .../04_Troubleshooting/TokenUsage/run.php | 94 +++++++++++++++++++ .../Requests/Traits/HandlesResponse.php | 62 ++++++------ src/ApiClient/Responses/ApiResponse.php | 2 + .../Responses/PartialApiResponse.php | 2 + src/ApiClient/Traits/HandlesApiResponse.php | 7 +- .../Traits/HandlesAsyncApiResponse.php | 8 +- .../Traits/HandlesStreamApiResponse.php | 20 ++-- .../Traits/PartialApiResponseReceived.php | 19 ++++ .../Anthropic/Traits/HandlesResponse.php | 34 +++---- src/Clients/Cohere/Traits/HandlesResponse.php | 28 +++--- src/Clients/Gemini/Traits/HandlesResponse.php | 28 +++--- src/Configs/Clients/AnthropicConfig.php | 2 +- src/Configs/Clients/AzureConfig.php | 2 +- src/Configs/Clients/CohereConfig.php | 2 +- src/Configs/Clients/FireworksConfig.php | 2 +- src/Configs/Clients/GeminiConfig.php | 2 +- src/Configs/Clients/GroqConfig.php | 2 +- src/Configs/Clients/MistralConfig.php | 2 +- src/Configs/Clients/OllamaConfig.php | 2 +- src/Configs/Clients/OpenAIConfig.php | 2 +- src/Configs/Clients/OpenRouterConfig.php | 2 +- src/Configs/Clients/TogetherConfig.php | 2 +- src/Events/ApiClient/ApiResponseReceived.php | 14 +-- ...guage.php => GuessProgrammingLanguage.php} | 2 +- ...hp => GuessProgrammingLanguageFromExt.php} | 2 +- .../Module/Modules/Markdown/SplitMarkdown.php | 1 - tests/MockLLM.php | 2 +- 38 files changed, 403 insertions(+), 203 deletions(-) delete mode 100644 docs/cookbook/examples/api_support/caching.mdx create mode 100644 docs/cookbook/examples/techniques/image_to_data_anthropic.mdx create mode 100644 docs/cookbook/examples/troubleshooting/token_usage_events.mdx create mode 100644 examples/04_Troubleshooting/TokenUsage/run.php create mode 100644 src/ApiClient/Traits/PartialApiResponseReceived.php rename src/Extras/Module/Modules/Code/{GuessLanguage.php => GuessProgrammingLanguage.php} (93%) rename src/Extras/Module/Modules/Code/{GuessLanguageFromExt.php => GuessProgrammingLanguageFromExt.php} (88%) diff --git a/README.md b/README.md index 69435f09..bde349dc 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,11 @@ Here's a simple CLI demo app using Instructor to extract structured data from te - OpenRouter support - access to 100+ language models - Use local models with Ollama +### Other capabilities + +- Developer friendly LLM context caching for reduced costs and faster inference (for Anthropic models) +- Developer friendly image processing (for OpenAI, Anthropic and Gemini models) + ### Documentation and examples - Learn more from growing documentation and 50+ cookbooks diff --git a/docs/cookbook/examples/api_support/caching.mdx b/docs/cookbook/examples/api_support/caching.mdx deleted file mode 100644 index f6c6dd94..00000000 --- a/docs/cookbook/examples/api_support/caching.mdx +++ /dev/null @@ -1,70 +0,0 @@ ---- -title: 'Fireworks.ai' -docname: 'fireworks' ---- - -## Overview - -Please note that the larger Mistral models support Mode::Json, which is much more -reliable than Mode::MdJson. - -Mode compatibility: -- Mode::Tools - selected models -- Mode::Json - selected models -- Mode::MdJson - - -## Example - -```php -add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); - -use Cognesy\Instructor\Clients\FireworksAI\FireworksAIClient; -use Cognesy\Instructor\Enums\Mode; -use Cognesy\Instructor\Instructor; -use Cognesy\Instructor\Utils\Env; - -enum UserType : string { - case Guest = 'guest'; - case User = 'user'; - case Admin = 'admin'; -} - -class User { - public int $age; - public string $name; - public string $username; - public UserType $role; - /** @var string[] */ - public array $hobbies; -} - -// Mistral instance params -$yourApiKey = Env::get('FIREWORKSAI_API_KEY'); // set your own API key - -// Create instance of client initialized with custom parameters -$client = new FireworksAIClient( - apiKey: $yourApiKey, -); - -/// Get Instructor with the default client component overridden with your own -$instructor = (new Instructor)->withClient($client); - -$user = $instructor - ->respond( - messages: "Jason (@jxnlco) is 25 years old and is the admin of this project. He likes playing football and reading books.", - responseModel: User::class, - model: 'accounts/fireworks/models/mixtral-8x7b-instruct', - mode: Mode::Json, - //options: ['stream' => true ] - ); - -print("Completed response model:\n\n"); -dump($user); - -assert(isset($user->name)); -assert(isset($user->age)); -?> -``` diff --git a/docs/cookbook/examples/basics/complex_extraction.mdx b/docs/cookbook/examples/basics/complex_extraction.mdx index 4b9b6be0..196bde1a 100644 --- a/docs/cookbook/examples/basics/complex_extraction.mdx +++ b/docs/cookbook/examples/basics/complex_extraction.mdx @@ -112,7 +112,7 @@ $events = $instructor ->request( messages: $report, responseModel: Sequence::of(ProjectEvent::class), - model: 'openai:gpt-4o-mini', + model: 'gpt-4o-mini', mode: Mode::Json, options: [ 'max_tokens' => 2048, diff --git a/docs/cookbook/examples/basics/complex_extraction_claude.mdx b/docs/cookbook/examples/basics/complex_extraction_claude.mdx index 88f5c0f4..3b086cad 100644 --- a/docs/cookbook/examples/basics/complex_extraction_claude.mdx +++ b/docs/cookbook/examples/basics/complex_extraction_claude.mdx @@ -16,7 +16,6 @@ $loader = require 'vendor/autoload.php'; $loader->add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); use Cognesy\Instructor\Clients\Anthropic\AnthropicClient; -use Cognesy\Instructor\Clients\OpenRouter\OpenRouterClient; use Cognesy\Instructor\Enums\Mode; use Cognesy\Instructor\Extras\Sequence\Sequence; use Cognesy\Instructor\Instructor; diff --git a/docs/cookbook/examples/techniques/image_to_data.mdx b/docs/cookbook/examples/techniques/image_to_data.mdx index c8ba1118..643ad10b 100644 --- a/docs/cookbook/examples/techniques/image_to_data.mdx +++ b/docs/cookbook/examples/techniques/image_to_data.mdx @@ -1,5 +1,5 @@ --- -title: 'Image to data' +title: 'Image to data (OpenAI)' docname: 'image_to_data' --- diff --git a/docs/cookbook/examples/techniques/image_to_data_anthropic.mdx b/docs/cookbook/examples/techniques/image_to_data_anthropic.mdx new file mode 100644 index 00000000..874f52bd --- /dev/null +++ b/docs/cookbook/examples/techniques/image_to_data_anthropic.mdx @@ -0,0 +1,74 @@ +--- +title: 'Image to data (Anthropic)' +docname: 'image_to_data_anthropic' +--- + +## Overview + +This is an example of how to extract structured data from an image using +Instructor. The image is loaded from a file and converted to base64 format +before sending it to OpenAI API. + +The response model is a PHP class that represents the structured receipt +information with data of vendor, items, subtotal, tax, tip, and total. + + +## Scanned image + +Here's the image we're going to extract data from. + +![Receipt](images/receipt.png) + + +## Example + +```php +add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); + +use Cognesy\Instructor\Clients\Anthropic\AnthropicClient; +use Cognesy\Instructor\Enums\Mode; +use Cognesy\Instructor\Extras\Image\Image; +use Cognesy\Instructor\Instructor; +use Cognesy\Instructor\Utils\Env; + +class Vendor { + public ?string $name = ''; + public ?string $address = ''; + public ?string $phone = ''; +} + +class ReceiptItem { + public string $name; + public ?int $quantity = 1; + public float $price; +} + +class Receipt { + public Vendor $vendor; + /** @var ReceiptItem[] */ + public array $items = []; + public ?float $subtotal; + public ?float $tax; + public ?float $tip; + public float $total; +} + +$client = new AnthropicClient( + apiKey: Env::get('ANTHROPIC_API_KEY'), +); + +$receipt = (new Instructor)->withClient($client)->respond( + input: Image::fromFile(__DIR__ . '/receipt.png'), + responseModel: Receipt::class, + prompt: 'Extract structured data from the receipt. Return result as JSON following this schema: <|json_schema|>', + mode: Mode::Json, + options: ['max_tokens' => 4096] +); + +dump($receipt); + +assert($receipt->total === 169.82); +?> +``` diff --git a/docs/cookbook/examples/troubleshooting/token_usage_events.mdx b/docs/cookbook/examples/troubleshooting/token_usage_events.mdx new file mode 100644 index 00000000..42ebb24d --- /dev/null +++ b/docs/cookbook/examples/troubleshooting/token_usage_events.mdx @@ -0,0 +1,94 @@ +--- +title: 'Tracking token usage via events' +docname: 'token_usage_events' +--- + +## Overview + +Some use cases require tracking the token usage of the API responses. +Currently, this can be done by listening to the `ApiResponseReceived` +and `PartialApiResponseReceived` events and summing the token usage +of the responses. + +Code below demonstrates how it can be implemented using Instructor +event listeners. + +> Note: OpenAI API requires `stream_options` to be set to +> `['include_usage' => true]` to include token usage in the streamed +> responses. + +## Example + +```php +add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); + +use Cognesy\Instructor\ApiClient\Responses\ApiResponse; +use Cognesy\Instructor\ApiClient\Responses\PartialApiResponse; +use Cognesy\Instructor\ApiClient\Traits\PartialApiResponseReceived; +use Cognesy\Instructor\Events\ApiClient\ApiResponseReceived; +use Cognesy\Instructor\Instructor; + +class User { + public int $age; + public string $name; +} + +class TokenCounter { + private int $input = 0; + private int $output = 0; + private int $cacheCreation = 0; + private int $cacheRead = 0; + + public function add(ApiResponse|PartialApiResponse $response) { + $this->input += $response->inputTokens; + $this->output += $response->outputTokens; + $this->cacheCreation += $response->cacheCreationTokens; + $this->cacheRead += $response->cacheReadTokens; + } + + public function reset() { + $this->input = 0; + $this->output = 0; + $this->cacheCreation = 0; + $this->cacheRead = 0; + } + + public function print() { + echo "Input tokens: $this->input\n"; + echo "Output tokens: $this->output\n"; + echo "Cache creation tokens: $this->cacheCreation\n"; + echo "Cache read tokens: $this->cacheRead\n"; + } +} + +$counter = new TokenCounter(); + +echo "COUNTING TOKENS FOR SYNC RESPONSE\n"; +$text = "Jason is 25 years old and works as an engineer."; +$instructor = (new Instructor) + ->onEvent(ApiResponseReceived::class, fn($e) => $counter->add($e->apiResponse)) + ->respond( + messages: $text, + responseModel: User::class, +); +echo "\nTEXT: $text\n"; +$counter->print(); +$counter->reset(); + +echo "\n\nCOUNTING TOKENS FOR STREAMED RESPONSE\n"; +$text = "Anna is 19 years old."; +$instructor = (new Instructor) + ->onEvent(PartialApiResponseReceived::class, fn($e) => $counter->add($e->partialApiResponse)) + ->respond( + messages: $text, + responseModel: User::class, + options: ['stream' => true, 'stream_options' => ['include_usage' => true]], +); +echo "\nTEXT: $text\n"; +$counter->print(); +$counter->reset(); + +?> +``` diff --git a/docs/mint.json b/docs/mint.json index b50b608b..d0cfa633 100644 --- a/docs/mint.json +++ b/docs/mint.json @@ -156,7 +156,7 @@ "cookbook/examples/techniques/entity_relationships", "cookbook/examples/techniques/handling_errors", "cookbook/examples/techniques/image_to_data", - "cookbook/examples/techniques/image_to_data", + "cookbook/examples/techniques/image_to_data_anthropic", "cookbook/examples/techniques/image_to_data", "cookbook/examples/techniques/limiting_lists", "cookbook/examples/techniques/restate_instructions", @@ -174,6 +174,7 @@ "cookbook/examples/troubleshooting/debugging", "cookbook/examples/troubleshooting/on_error", "cookbook/examples/troubleshooting/on_event", + "cookbook/examples/troubleshooting/token_usage_events", "cookbook/examples/troubleshooting/wiretap" ] }, diff --git a/examples/01_Basics/ComplexExtraction/run.php b/examples/01_Basics/ComplexExtraction/run.php index 4b9b6be0..196bde1a 100644 --- a/examples/01_Basics/ComplexExtraction/run.php +++ b/examples/01_Basics/ComplexExtraction/run.php @@ -112,7 +112,7 @@ enum StakeholderRole: string { ->request( messages: $report, responseModel: Sequence::of(ProjectEvent::class), - model: 'openai:gpt-4o-mini', + model: 'gpt-4o-mini', mode: Mode::Json, options: [ 'max_tokens' => 2048, diff --git a/examples/03_Techniques/ImageToData/run.php b/examples/03_Techniques/ImageToData/run.php index c8ba1118..643ad10b 100644 --- a/examples/03_Techniques/ImageToData/run.php +++ b/examples/03_Techniques/ImageToData/run.php @@ -1,5 +1,5 @@ --- -title: 'Image to data' +title: 'Image to data (OpenAI)' docname: 'image_to_data' --- diff --git a/examples/03_Techniques/ImageToDataAnthropic/run.php b/examples/03_Techniques/ImageToDataAnthropic/run.php index 3936075b..874f52bd 100644 --- a/examples/03_Techniques/ImageToDataAnthropic/run.php +++ b/examples/03_Techniques/ImageToDataAnthropic/run.php @@ -1,6 +1,6 @@ --- -title: 'Image to data' -docname: 'image_to_data' +title: 'Image to data (Anthropic)' +docname: 'image_to_data_anthropic' --- ## Overview diff --git a/examples/04_Troubleshooting/TokenUsage/run.php b/examples/04_Troubleshooting/TokenUsage/run.php new file mode 100644 index 00000000..42ebb24d --- /dev/null +++ b/examples/04_Troubleshooting/TokenUsage/run.php @@ -0,0 +1,94 @@ +--- +title: 'Tracking token usage via events' +docname: 'token_usage_events' +--- + +## Overview + +Some use cases require tracking the token usage of the API responses. +Currently, this can be done by listening to the `ApiResponseReceived` +and `PartialApiResponseReceived` events and summing the token usage +of the responses. + +Code below demonstrates how it can be implemented using Instructor +event listeners. + +> Note: OpenAI API requires `stream_options` to be set to +> `['include_usage' => true]` to include token usage in the streamed +> responses. + +## Example + +```php +add('Cognesy\\Instructor\\', __DIR__ . '../../src/'); + +use Cognesy\Instructor\ApiClient\Responses\ApiResponse; +use Cognesy\Instructor\ApiClient\Responses\PartialApiResponse; +use Cognesy\Instructor\ApiClient\Traits\PartialApiResponseReceived; +use Cognesy\Instructor\Events\ApiClient\ApiResponseReceived; +use Cognesy\Instructor\Instructor; + +class User { + public int $age; + public string $name; +} + +class TokenCounter { + private int $input = 0; + private int $output = 0; + private int $cacheCreation = 0; + private int $cacheRead = 0; + + public function add(ApiResponse|PartialApiResponse $response) { + $this->input += $response->inputTokens; + $this->output += $response->outputTokens; + $this->cacheCreation += $response->cacheCreationTokens; + $this->cacheRead += $response->cacheReadTokens; + } + + public function reset() { + $this->input = 0; + $this->output = 0; + $this->cacheCreation = 0; + $this->cacheRead = 0; + } + + public function print() { + echo "Input tokens: $this->input\n"; + echo "Output tokens: $this->output\n"; + echo "Cache creation tokens: $this->cacheCreation\n"; + echo "Cache read tokens: $this->cacheRead\n"; + } +} + +$counter = new TokenCounter(); + +echo "COUNTING TOKENS FOR SYNC RESPONSE\n"; +$text = "Jason is 25 years old and works as an engineer."; +$instructor = (new Instructor) + ->onEvent(ApiResponseReceived::class, fn($e) => $counter->add($e->apiResponse)) + ->respond( + messages: $text, + responseModel: User::class, +); +echo "\nTEXT: $text\n"; +$counter->print(); +$counter->reset(); + +echo "\n\nCOUNTING TOKENS FOR STREAMED RESPONSE\n"; +$text = "Anna is 19 years old."; +$instructor = (new Instructor) + ->onEvent(PartialApiResponseReceived::class, fn($e) => $counter->add($e->partialApiResponse)) + ->respond( + messages: $text, + responseModel: User::class, + options: ['stream' => true, 'stream_options' => ['include_usage' => true]], +); +echo "\nTEXT: $text\n"; +$counter->print(); +$counter->reset(); + +?> +``` diff --git a/src/ApiClient/Requests/Traits/HandlesResponse.php b/src/ApiClient/Requests/Traits/HandlesResponse.php index 064ce9d2..1582943c 100644 --- a/src/ApiClient/Requests/Traits/HandlesResponse.php +++ b/src/ApiClient/Requests/Traits/HandlesResponse.php @@ -15,48 +15,52 @@ public function toApiResponse(Response $response): ApiResponse { if (empty($decoded)) { throw new Exception('Response body empty or does not contain correct JSON: ' . $response->body()); } - $finishReason = $decoded['choices'][0]['finish_reason'] ?? ''; - $toolName = $decoded['choices'][0]['message']['tool_calls'][0]['function']['name'] ?? ''; - $contentMsg = $decoded['choices'][0]['message']['content'] ?? ''; - $contentFnArgs = $decoded['choices'][0]['message']['tool_calls'][0]['function']['arguments'] ?? ''; - $content = match(true) { - !empty($contentMsg) => $contentMsg, - !empty($contentFnArgs) => $contentFnArgs, - default => '' - }; - $inputTokens = $decoded['usage']['prompt_tokens'] ?? 0; - $outputTokens = $decoded['usage']['completion_tokens'] ?? 0; return new ApiResponse( - content: $content, + content: $this->getContent($decoded), responseData: $decoded, - toolName: $toolName, - finishReason: $finishReason, + toolName: $decoded['choices'][0]['message']['tool_calls'][0]['function']['name'] ?? '', + finishReason: $decoded['choices'][0]['finish_reason'] ?? '', toolCalls: null, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + inputTokens: $decoded['usage']['prompt_tokens'] ?? 0, + outputTokens: $decoded['usage']['completion_tokens'] ?? 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, ); } public function toPartialApiResponse(string $partialData) : PartialApiResponse { $decoded = Json::parse($partialData, default: []); - $finishReason = $decoded['choices'][0]['finish_reason'] ?? ''; - $toolName = $decoded['choices'][0]['delta']['tool_calls'][0]['function']['name'] ?? ''; + return new PartialApiResponse( + delta: $this->getDelta($decoded), + responseData: $decoded, + toolName: $decoded['choices'][0]['delta']['tool_calls'][0]['function']['name'] ?? '', + finishReason: $decoded['choices'][0]['finish_reason'] ?? '', + inputTokens: $decoded['usage']['prompt_tokens'] ?? 0, + outputTokens: $decoded['usage']['completion_tokens'] ?? 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, + ); + } + + // INTERNAL /////////////////////////////////////////////////////////////////////////////////////// + + private function getContent(array $decoded): string { + $contentMsg = $decoded['choices'][0]['message']['content'] ?? ''; + $contentFnArgs = $decoded['choices'][0]['message']['tool_calls'][0]['function']['arguments'] ?? ''; + return match(true) { + !empty($contentMsg) => $contentMsg, + !empty($contentFnArgs) => $contentFnArgs, + default => '' + }; + } + + private function getDelta(array $decoded): string { $deltaContent = $decoded['choices'][0]['delta']['content'] ?? ''; $deltaFnArgs = $decoded['choices'][0]['delta']['tool_calls'][0]['function']['arguments'] ?? ''; - $delta = match(true) { + return match(true) { !empty($deltaContent) => $deltaContent, !empty($deltaFnArgs) => $deltaFnArgs, default => '' }; - $inputTokens = $decoded['usage']['prompt_tokens'] ?? 0; - $outputTokens = $decoded['usage']['completion_tokens'] ?? 0; - return new PartialApiResponse( - delta: $delta, - responseData: $decoded, - toolName: $toolName, - finishReason: $finishReason, - inputTokens: $inputTokens, - outputTokens: $outputTokens, - ); } } \ No newline at end of file diff --git a/src/ApiClient/Responses/ApiResponse.php b/src/ApiClient/Responses/ApiResponse.php index 5f2749e3..c1ed91e6 100644 --- a/src/ApiClient/Responses/ApiResponse.php +++ b/src/ApiClient/Responses/ApiResponse.php @@ -15,6 +15,8 @@ public function __construct( public ?ToolCalls $toolCalls = null, public int $inputTokens = 0, public int $outputTokens = 0, + public int $cacheCreationTokens = 0, + public int $cacheReadTokens = 0, ) {} public function getJson(): string { diff --git a/src/ApiClient/Responses/PartialApiResponse.php b/src/ApiClient/Responses/PartialApiResponse.php index 0de5522f..e6dcb96b 100644 --- a/src/ApiClient/Responses/PartialApiResponse.php +++ b/src/ApiClient/Responses/PartialApiResponse.php @@ -11,5 +11,7 @@ public function __construct( public string $finishReason = '', public int $inputTokens = 0, public int $outputTokens = 0, + public int $cacheCreationTokens = 0, + public int $cacheReadTokens = 0, ) {} } \ No newline at end of file diff --git a/src/ApiClient/Traits/HandlesApiResponse.php b/src/ApiClient/Traits/HandlesApiResponse.php index 7b6d4127..1028f8cf 100644 --- a/src/ApiClient/Traits/HandlesApiResponse.php +++ b/src/ApiClient/Traits/HandlesApiResponse.php @@ -3,7 +3,6 @@ use Cognesy\Instructor\ApiClient\Requests\ApiRequest; use Cognesy\Instructor\ApiClient\Responses\ApiResponse; -use Cognesy\Instructor\ApiClient\Utils\Debugger; use Cognesy\Instructor\Events\ApiClient\ApiRequestErrorRaised; use Cognesy\Instructor\Events\ApiClient\ApiRequestInitiated; use Cognesy\Instructor\Events\ApiClient\ApiRequestSent; @@ -25,11 +24,7 @@ public function get() : ApiResponse { $response = $this->respondRaw($request); $apiResponse = $this->apiRequest->toApiResponse($response); - $this->events->dispatch(new ApiResponseReceived( - $response->status(), - $this->getRequestHeaders($response), - $apiResponse->content, - )); + $this->events->dispatch(new ApiResponseReceived($apiResponse)); $this->tryDebug($request, $response, $apiResponse->content); diff --git a/src/ApiClient/Traits/HandlesAsyncApiResponse.php b/src/ApiClient/Traits/HandlesAsyncApiResponse.php index 6ba7bdfc..9027f3dc 100644 --- a/src/ApiClient/Traits/HandlesAsyncApiResponse.php +++ b/src/ApiClient/Traits/HandlesAsyncApiResponse.php @@ -32,11 +32,15 @@ public function async() : PromiseInterface { } protected function asyncRaw(ApiRequest $request, callable $onSuccess = null, callable $onError = null) : PromiseInterface { - $this?->events->dispatch(new ApiAsyncRequestInitiated($request)); + $this?->events->dispatch(new ApiAsyncRequestInitiated($request->toArray())); $promise = $this->connector()->sendAsync($request); if (!empty($onSuccess)) { $promise->then(function (Response $response) use ($onSuccess) { - $this?->events->dispatch(new ApiAsyncResponseReceived($response)); + $this?->events->dispatch(new ApiAsyncResponseReceived( + $response->status(), + $this->getRequestHeaders($response), + $response->content(), + )); $onSuccess($response); }); } diff --git a/src/ApiClient/Traits/HandlesStreamApiResponse.php b/src/ApiClient/Traits/HandlesStreamApiResponse.php index 5d65bbe7..59c0d062 100644 --- a/src/ApiClient/Traits/HandlesStreamApiResponse.php +++ b/src/ApiClient/Traits/HandlesStreamApiResponse.php @@ -2,19 +2,15 @@ namespace Cognesy\Instructor\ApiClient\Traits; -use Cognesy\Instructor\ApiClient\Requests\ApiRequest; use Cognesy\Instructor\ApiClient\Responses\PartialApiResponse; -use Cognesy\Instructor\ApiClient\Utils\Debugger; use Cognesy\Instructor\Events\ApiClient\ApiRequestErrorRaised; use Cognesy\Instructor\Events\ApiClient\ApiStreamConnected; use Cognesy\Instructor\Events\ApiClient\ApiStreamRequestInitiated; use Cognesy\Instructor\Events\ApiClient\ApiStreamRequestSent; -use Cognesy\Instructor\Events\ApiClient\ApiStreamResponseReceived; +//use Cognesy\Instructor\Events\ApiClient\ApiStreamResponseReceived; use Cognesy\Instructor\Events\ApiClient\ApiStreamUpdateReceived; use Exception; use Generator; -use Saloon\Exceptions\Request\RequestException; -use Saloon\Http\Response; trait HandlesStreamApiResponse { @@ -27,7 +23,9 @@ public function stream() : Generator { if (empty($response) || $this->isDone($response)) { continue; } - yield $this->apiRequest->toPartialApiResponse($response); + $partialApiResponse = $this->apiRequest->toPartialApiResponse($response); + $this->events->dispatch(new PartialApiResponseReceived($partialApiResponse)); + yield $partialApiResponse; } } @@ -73,11 +71,11 @@ protected function streamRaw(): Generator { yield $streamedData; } - $this?->events->dispatch(new ApiStreamResponseReceived( - $response->status(), - $this->getResponseHeaders($response), - $body, - )); +// $this?->events->dispatch(new ApiStreamResponseReceived( +// $response->status(), +// $this->getResponseHeaders($response), +// $body, +// )); $this->tryDebug($request, $response, $body); } diff --git a/src/ApiClient/Traits/PartialApiResponseReceived.php b/src/ApiClient/Traits/PartialApiResponseReceived.php new file mode 100644 index 00000000..23a44f07 --- /dev/null +++ b/src/ApiClient/Traits/PartialApiResponseReceived.php @@ -0,0 +1,19 @@ +partialApiResponse); + } +} diff --git a/src/Clients/Anthropic/Traits/HandlesResponse.php b/src/Clients/Anthropic/Traits/HandlesResponse.php index e5925615..478bb4e9 100644 --- a/src/Clients/Anthropic/Traits/HandlesResponse.php +++ b/src/Clients/Anthropic/Traits/HandlesResponse.php @@ -1,5 +1,4 @@ body()); - $content = $decoded['content'][0]['text'] ?? Json::encode($decoded['content'][0]['input']) ?? ''; - $toolName = $decoded['content'][0]['name'] ?? ''; - $finishReason = $decoded['stop_reason'] ?? ''; - $inputTokens = $decoded['usage']['input_tokens'] ?? 0; - $outputTokens = $decoded['usage']['output_tokens'] ?? 0; return new ApiResponse( - content: $content, + content: $decoded['content'][0]['text'] ?? Json::encode($decoded['content'][0]['input']) ?? '', responseData: $decoded, - toolName: $toolName, - finishReason: $finishReason, + toolName: $decoded['content'][0]['name'] ?? '', + finishReason: $decoded['stop_reason'] ?? '', toolCalls: null, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + inputTokens: $decoded['usage']['input_tokens'] ?? 0, + outputTokens: $decoded['usage']['output_tokens'] ?? 0, + cacheCreationTokens: $decoded['usage']['cache_creation_input_tokens'] ?? 0, + cacheReadTokens: $decoded['usage']['cache_read_input_tokens'] ?? 0, ); } public function toPartialApiResponse(string $partialData) : PartialApiResponse { $decoded = Json::parse($partialData, default: []); - $delta = $decoded['delta']['text'] ?? $decoded['delta']['partial_json'] ?? ''; - $toolName = $decoded['content_block']['name'] ?? ''; - $inputTokens = $decoded['message']['usage']['input_tokens'] ?? $decoded['usage']['input_tokens'] ?? 0; - $outputTokens = $decoded['message']['usage']['output_tokens'] ?? $decoded['usage']['output_tokens'] ?? 0; $finishReason = $decoded['message']['stop_reason'] ?? $decoded['message']['stop_reason'] ?? ''; return new PartialApiResponse( - delta: $delta, + delta: $decoded['delta']['text'] ?? $decoded['delta']['partial_json'] ?? '', responseData: $decoded, - toolName: $toolName, + toolName: $decoded['content_block']['name'] ?? '', finishReason: $finishReason, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + inputTokens: $decoded['message']['usage']['input_tokens'] ?? $decoded['usage']['input_tokens'] ?? 0, + outputTokens: $decoded['message']['usage']['output_tokens'] ?? $decoded['usage']['output_tokens'] ?? 0, + cacheCreationTokens: $decoded['message']['usage']['cache_creation_input_tokens'] ?? $decoded['usage']['cache_creation_input_tokens'] ?? 0, + cacheReadTokens: $decoded['message']['usage']['cache_read_input_tokens'] ?? $decoded['usage']['cache_read_input_tokens'] ?? 0, ); } -} \ No newline at end of file +} diff --git a/src/Clients/Cohere/Traits/HandlesResponse.php b/src/Clients/Cohere/Traits/HandlesResponse.php index 133171b0..b9277d37 100644 --- a/src/Clients/Cohere/Traits/HandlesResponse.php +++ b/src/Clients/Cohere/Traits/HandlesResponse.php @@ -10,34 +10,30 @@ trait HandlesResponse { public function toApiResponse(Response $response): ApiResponse { $decoded = Json::parse($response->body()); - $content = $decoded['text'] ?? ''; - $finishReason = $decoded['finish_reason'] ?? ''; - $inputTokens = $decoded['meta']['tokens']['input_tokens'] ?? 0; - $outputTokens = $decoded['meta']['tokens']['output_tokens'] ?? 0; return new ApiResponse( - content: $content, + content: $decoded['text'] ?? '', responseData: $decoded, toolName: '', - finishReason: $finishReason, + finishReason: $decoded['finish_reason'] ?? '', toolCalls: null, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + inputTokens: $decoded['meta']['tokens']['input_tokens'] ?? 0, + outputTokens: $decoded['meta']['tokens']['output_tokens'] ?? 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, ); } public function toPartialApiResponse(string $partialData) : PartialApiResponse { $decoded = Json::parse($partialData, default: []); - $delta = $decoded['text'] ?? $decoded['tool_calls'][0]['parameters'] ?? ''; - $inputTokens = $decoded['message']['usage']['input_tokens'] ?? $decoded['usage']['input_tokens'] ?? 0; - $outputTokens = $decoded['message']['usage']['output_tokens'] ?? $decoded['usage']['input_tokens'] ?? 0; - $finishReason = $decoded['finish_reason'] ?? ''; return new PartialApiResponse( - delta: $delta, + delta: $decoded['text'] ?? $decoded['tool_calls'][0]['parameters'] ?? '', responseData: $decoded, toolName: $decoded['tool_calls'][0]['name'] ?? '', - finishReason: $finishReason, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + finishReason: $decoded['finish_reason'] ?? '', + inputTokens: $decoded['message']['usage']['input_tokens'] ?? $decoded['usage']['input_tokens'] ?? 0, + outputTokens: $decoded['message']['usage']['output_tokens'] ?? $decoded['usage']['input_tokens'] ?? 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, ); } } \ No newline at end of file diff --git a/src/Clients/Gemini/Traits/HandlesResponse.php b/src/Clients/Gemini/Traits/HandlesResponse.php index 7e8050ad..bb80b988 100644 --- a/src/Clients/Gemini/Traits/HandlesResponse.php +++ b/src/Clients/Gemini/Traits/HandlesResponse.php @@ -10,34 +10,30 @@ trait HandlesResponse { public function toApiResponse(Response $response): ApiResponse { $decoded = Json::parse($response->body()); - $content = $decoded['candidates'][0]['content']['parts'][0]['text'] ?? ''; - $finishReason = $decoded['candidates'][0]['finishReason'] ?? ''; - $inputTokens = $decoded['usageMetadata']['promptTokenCount'] ?? 0; - $outputTokens = $decoded['usageMetadata']['candidatesTokenCount'] ?? 0; return new ApiResponse( - content: $content, + content: $decoded['candidates'][0]['content']['parts'][0]['text'] ?? '', responseData: $decoded, toolName: '', - finishReason: $finishReason, + finishReason: $decoded['candidates'][0]['finishReason'] ?? '', toolCalls: null, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + inputTokens: $decoded['usageMetadata']['promptTokenCount'] ?? 0, + outputTokens: $decoded['usageMetadata']['candidatesTokenCount'] ?? 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, ); } public function toPartialApiResponse(string $partialData) : PartialApiResponse { $decoded = Json::parse($partialData, default: []); - $delta = $decoded['candidates'][0]['content']['parts'][0]['text'] ?? ''; - $inputTokens = $decoded['usageMetadata']['promptTokenCount'] ?? 0; - $outputTokens = $decoded['usageMetadata']['candidatesTokenCount'] ?? 0; - $finishReason = $decoded['candidates'][0]['finishReason'] ?? ''; return new PartialApiResponse( - delta: $delta, + delta: $decoded['candidates'][0]['content']['parts'][0]['text'] ?? '', responseData: $decoded, toolName: $decoded['tool_calls'][0]['name'] ?? '', - finishReason: $finishReason, - inputTokens: $inputTokens, - outputTokens: $outputTokens, + finishReason: $decoded['candidates'][0]['finishReason'] ?? '', + inputTokens: $decoded['usageMetadata']['promptTokenCount'] ?? 0, + outputTokens: $decoded['usageMetadata']['candidatesTokenCount'] ?? 0, + cacheCreationTokens: 0, + cacheReadTokens: 0, ); } } diff --git a/src/Configs/Clients/AnthropicConfig.php b/src/Configs/Clients/AnthropicConfig.php index 18fcf15b..699a152b 100644 --- a/src/Configs/Clients/AnthropicConfig.php +++ b/src/Configs/Clients/AnthropicConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Anthropic\AnthropicClient; use Cognesy\Instructor\Clients\Anthropic\AnthropicConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/AzureConfig.php b/src/Configs/Clients/AzureConfig.php index 369f7a65..3cd0b4b6 100644 --- a/src/Configs/Clients/AzureConfig.php +++ b/src/Configs/Clients/AzureConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Azure\AzureClient; use Cognesy\Instructor\Clients\Azure\AzureConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/CohereConfig.php b/src/Configs/Clients/CohereConfig.php index 81f1df36..f0ff9949 100644 --- a/src/Configs/Clients/CohereConfig.php +++ b/src/Configs/Clients/CohereConfig.php @@ -2,7 +2,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Cohere\CohereClient; use Cognesy\Instructor\Clients\Cohere\CohereConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/FireworksConfig.php b/src/Configs/Clients/FireworksConfig.php index ffcf7f37..2f332478 100644 --- a/src/Configs/Clients/FireworksConfig.php +++ b/src/Configs/Clients/FireworksConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\FireworksAI\FireworksAIClient; use Cognesy\Instructor\Clients\FireworksAI\FireworksAIConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/GeminiConfig.php b/src/Configs/Clients/GeminiConfig.php index 5bb2004d..7cdabd2c 100644 --- a/src/Configs/Clients/GeminiConfig.php +++ b/src/Configs/Clients/GeminiConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Gemini\GeminiClient; use Cognesy\Instructor\Clients\Gemini\GeminiConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/GroqConfig.php b/src/Configs/Clients/GroqConfig.php index 10024ff3..79b87829 100644 --- a/src/Configs/Clients/GroqConfig.php +++ b/src/Configs/Clients/GroqConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Groq\GroqClient; use Cognesy\Instructor\Clients\Groq\GroqConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/MistralConfig.php b/src/Configs/Clients/MistralConfig.php index f9f28d30..a09152b8 100644 --- a/src/Configs/Clients/MistralConfig.php +++ b/src/Configs/Clients/MistralConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Mistral\MistralClient; use Cognesy\Instructor\Clients\Mistral\MistralConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/OllamaConfig.php b/src/Configs/Clients/OllamaConfig.php index 0285e8f3..81184b3d 100644 --- a/src/Configs/Clients/OllamaConfig.php +++ b/src/Configs/Clients/OllamaConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\Ollama\OllamaClient; use Cognesy\Instructor\Clients\Ollama\OllamaConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/OpenAIConfig.php b/src/Configs/Clients/OpenAIConfig.php index 89d67de9..35b42a4d 100644 --- a/src/Configs/Clients/OpenAIConfig.php +++ b/src/Configs/Clients/OpenAIConfig.php @@ -4,7 +4,7 @@ use Cognesy\Instructor\ApiClient\Contracts\CanCallApi; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\OpenAI\OpenAIClient; use Cognesy\Instructor\Clients\OpenAI\OpenAIConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/OpenRouterConfig.php b/src/Configs/Clients/OpenRouterConfig.php index e3df4f3a..ed958c4a 100644 --- a/src/Configs/Clients/OpenRouterConfig.php +++ b/src/Configs/Clients/OpenRouterConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\OpenRouter\OpenRouterClient; use Cognesy\Instructor\Clients\OpenRouter\OpenRouterConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Configs/Clients/TogetherConfig.php b/src/Configs/Clients/TogetherConfig.php index fa02df18..eb0fafbb 100644 --- a/src/Configs/Clients/TogetherConfig.php +++ b/src/Configs/Clients/TogetherConfig.php @@ -3,7 +3,7 @@ namespace Cognesy\Instructor\Configs\Clients; use Cognesy\Instructor\ApiClient\Factories\ApiRequestFactory; -use Cognesy\Instructor\ApiClient\ModelParams; +//use Cognesy\Instructor\ApiClient\ModelParams; use Cognesy\Instructor\Clients\TogetherAI\TogetherAIClient; use Cognesy\Instructor\Clients\TogetherAI\TogetherAIConnector; use Cognesy\Instructor\Configuration\Configuration; diff --git a/src/Events/ApiClient/ApiResponseReceived.php b/src/Events/ApiClient/ApiResponseReceived.php index d51b07a1..4511951f 100644 --- a/src/Events/ApiClient/ApiResponseReceived.php +++ b/src/Events/ApiClient/ApiResponseReceived.php @@ -1,25 +1,19 @@ $this->status, - 'headers' => $this->headers, - 'body' => $this->body, - ]); + return Json::encode($this->apiResponse); } -} \ No newline at end of file +} diff --git a/src/Extras/Module/Modules/Code/GuessLanguage.php b/src/Extras/Module/Modules/Code/GuessProgrammingLanguage.php similarity index 93% rename from src/Extras/Module/Modules/Code/GuessLanguage.php rename to src/Extras/Module/Modules/Code/GuessProgrammingLanguage.php index f12fa07a..58b72a05 100644 --- a/src/Extras/Module/Modules/Code/GuessLanguage.php +++ b/src/Extras/Module/Modules/Code/GuessProgrammingLanguage.php @@ -11,7 +11,7 @@ #[ModuleSignature('code -> language')] #[ModuleDescription("Identify the programming language of the code. Return only the language name.")] -class GuessLanguage extends Prediction +class GuessProgrammingLanguage extends Prediction { // private Predictor $guessLanguage; // diff --git a/src/Extras/Module/Modules/Code/GuessLanguageFromExt.php b/src/Extras/Module/Modules/Code/GuessProgrammingLanguageFromExt.php similarity index 88% rename from src/Extras/Module/Modules/Code/GuessLanguageFromExt.php rename to src/Extras/Module/Modules/Code/GuessProgrammingLanguageFromExt.php index a91c3dff..6eb6aa73 100644 --- a/src/Extras/Module/Modules/Code/GuessLanguageFromExt.php +++ b/src/Extras/Module/Modules/Code/GuessProgrammingLanguageFromExt.php @@ -5,7 +5,7 @@ use Cognesy\Instructor\Extras\Module\Core\Module; use Cognesy\Instructor\Extras\Module\Modules\Code\Enums\Language; -class GuessLanguageFromExt extends Module +class GuessProgrammingLanguageFromExt extends Module { public function for(string $filePath) : string { return ($this)(filePath: $filePath)->get('language'); diff --git a/src/Extras/Module/Modules/Markdown/SplitMarkdown.php b/src/Extras/Module/Modules/Markdown/SplitMarkdown.php index 39624dcc..cd1acfda 100644 --- a/src/Extras/Module/Modules/Markdown/SplitMarkdown.php +++ b/src/Extras/Module/Modules/Markdown/SplitMarkdown.php @@ -1,5 +1,4 @@ shouldReceive('createApiRequest')->andReturnUsing(fn() => new OpenAIApiRequest()); $mockLLM->shouldReceive('getApiRequest')->andReturnUsing(fn() => new OpenAIApiRequest()); - $mockLLM->shouldReceive('defaultModel')->andReturn('openai:gpt-4o'); + $mockLLM->shouldReceive('defaultModel')->andReturn('gpt-4o-mini'); $mockLLM->shouldReceive('defaultMaxTokens')->andReturn('1024'); $mockLLM->shouldReceive('getModeRequestClass')->andReturn(OpenAIApiRequest::class); $mockLLM->shouldReceive('get')->andReturnUsing(...$list);