Bug fixes in LLM connectivity

cognesy · Oct 8, 2024 · 02e8885 · 02e8885
1 parent 5586867
commit 02e8885
Show file tree

Hide file tree

Showing 6 changed files with 21 additions and 17 deletions.
diff --git a/config/llm.php b/config/llm.php
@@ -111,7 +111,7 @@
         ],
         'openrouter' => [
             'providerType' => LLMProviderType::OpenRouter->value,
-            'apiUrl' => 'https://openrouter.ai/api/v1/',
+            'apiUrl' => 'https://openrouter.ai/api/v1',
             'apiKey' => Env::get('OPENROUTER_API_KEY', ''),
             'endpoint' => '/chat/completions',
             'defaultModel' => 'qwen/qwen-2.5-72b-instruct', //'microsoft/phi-3.5-mini-128k-instruct',

diff --git a/evals/LLMModes/run.php b/evals/LLMModes/run.php
@@ -12,6 +12,7 @@
 $connections = [
     'azure',
     'cohere1',
+    'cohere2',
     'fireworks',
     'gemini',
     'groq',

diff --git a/src/Features/Http/Drivers/GuzzleDriver.php b/src/Features/Http/Drivers/GuzzleDriver.php
@@ -35,10 +35,11 @@ public function __construct(
         if (isset($this->httpClient) && Debug::isEnabled()) {
             throw new InvalidArgumentException("Guzzle does not allow to inject debugging stack into existing client. Turn off debug or use default client.");
         }
-        $this->client = match(Debug::isEnabled()) {
-            false => $httpClient ?? new Client(),
-            true => new Client(['handler' => $this->addDebugStack(HandlerStack::create())]),
-        };
+//        $this->client = match(Debug::isEnabled()) {
+//            false => $httpClient ?? new Client(),
+//            true => new Client(['handler' => $this->addDebugStack(HandlerStack::create())]),
+//        };
+        $this->client = new Client(['handler' => $this->addDebugStack(HandlerStack::create())]);
     }
 
     public function handle(

diff --git a/src/Features/LLM/Drivers/CohereV2Driver.php b/src/Features/LLM/Drivers/CohereV2Driver.php
@@ -57,7 +57,7 @@ public function toLLMResponse(array $data): LLMResponse {
         );
     }
 
-    public function toPartialLLMResponse(array $data) : ?PartialLLMResponse {
+    public function toPartialLLMResponse(array|null $data) : ?PartialLLMResponse {
         if (empty($data)) {
             return null;
         }
@@ -80,7 +80,7 @@ public function getData(string $data): string|bool {
         }
         $data = trim(substr($data, 5));
         return match(true) {
-            $data === '"[DONE]"' => false,
+            $data === '[DONE]' => false,
             default => $data,
         };
     }

diff --git a/src/Features/LLM/Drivers/OpenAIDriver.php b/src/Features/LLM/Drivers/OpenAIDriver.php
@@ -99,8 +99,8 @@ public function toLLMResponse(array $data): ?LLMResponse {
         );
     }
 
-    public function toPartialLLMResponse(array $data) : ?PartialLLMResponse {
-        if (empty($data)) {
+    public function toPartialLLMResponse(array|null $data) : ?PartialLLMResponse {
+        if ($data === null || empty($data)) {
             return null;
         }
         return new PartialLLMResponse(

diff --git a/src/Features/LLM/InferenceResponse.php b/src/Features/LLM/InferenceResponse.php
@@ -80,19 +80,21 @@ public function asPartialLLMResponses() : Generator {
             if ($streamEvent === false) {
                 continue;
             }
-            $result = $this->driver->toPartialLLMResponse(Json::decode($streamEvent));
-            if ($result === null) {
+            $data = Json::decode($streamEvent, []);
+            $partialResponse = $this->driver->toPartialLLMResponse($data);
+            if ($partialResponse === null) {
                 continue;
             }
-            if ($result->finishReason !== '') {
-                $finishReason = $result->finishReason;
+            if ($partialResponse->finishReason !== '') {
+                $finishReason = $partialResponse->finishReason;
             }
-            $content .= $result->contentDelta;
-            $partialResponse = $result
+            $content .= $partialResponse->contentDelta;
+            // add accumulated content and last finish reason
+            $enrichedResponse = $partialResponse
                 ->withContent($content)
                 ->withFinishReason($finishReason);
-            $this->events->dispatch(new PartialLLMResponseReceived($partialResponse));
-            yield $partialResponse;
+            $this->events->dispatch(new PartialLLMResponseReceived($enrichedResponse));
+            yield $enrichedResponse;
         }
     }