From 07baf4c2ed2f77e9033e3be24ea351d270acd840 Mon Sep 17 00:00:00 2001
From: "github-actions[bot]"
 <41898282+github-actions[bot]@users.noreply.github.com>
Date: Mon, 27 Jan 2025 20:43:33 +0000
Subject: [PATCH] fix(llmobs): fix token extraction for chat completion streams
 [backport 2.20] (#12091)

Backport 75179ef6425d0318efb1f6c8aa31e072bc620c61 from #12070 to 2.20.

Fixes token chunk extraction to account for the `choices` field in a
chunk being an empty list

#### Before
```
Error generating LLMObs span event for span <Span(id=16151817411339450163,trace_id=137677390470467884790869841527646927357,parent_id=None,name=openai.request)>, likely due to malformed span
Traceback (most recent call last):
  File "/XXXXX/ddtrace/contrib/internal/openai/utils.py", line 118, in __aiter__
    await self._extract_token_chunk(chunk)
  File "/XXXXX/ddtrace/contrib/internal/openai/utils.py", line 157, in _extract_token_chunk
    choice = getattr(chunk, "choices", [None])[0]
IndexError: list index out of range
```

#### After
Traced succesfully
<img width="904" alt="image"
src="https://github.com/user-attachments/assets/43c68edd-03f7-4105-a3d3-213eeb5fb0ab"
/>


Co-authored-by: lievan <42917263+lievan@users.noreply.github.com>
Co-authored-by: kyle <kyle@verhoog.ca>
---
 ddtrace/contrib/internal/openai/utils.py               | 10 ++++++++--
 .../notes/fix-token-extraction-0133808742374ef4.yaml   |  4 ++++
 2 files changed, 12 insertions(+), 2 deletions(-)
 create mode 100644 releasenotes/notes/fix-token-extraction-0133808742374ef4.yaml

diff --git a/ddtrace/contrib/internal/openai/utils.py b/ddtrace/contrib/internal/openai/utils.py
index f5dfc10efef..0217b1e61d2 100644
--- a/ddtrace/contrib/internal/openai/utils.py
+++ b/ddtrace/contrib/internal/openai/utils.py
@@ -89,7 +89,10 @@ def _extract_token_chunk(self, chunk):
         """Attempt to extract the token chunk (last chunk in the stream) from the streamed response."""
         if not self._dd_span._get_ctx_item("_dd.auto_extract_token_chunk"):
             return
-        choice = getattr(chunk, "choices", [None])[0]
+        choices = getattr(chunk, "choices")
+        if not choices:
+            return
+        choice = choices[0]
         if not getattr(choice, "finish_reason", None):
             # Only the second-last chunk in the stream with token usage enabled will have finish_reason set
             return
@@ -152,7 +155,10 @@ async def _extract_token_chunk(self, chunk):
         """Attempt to extract the token chunk (last chunk in the stream) from the streamed response."""
         if not self._dd_span._get_ctx_item("_dd.auto_extract_token_chunk"):
             return
-        choice = getattr(chunk, "choices", [None])[0]
+        choices = getattr(chunk, "choices")
+        if not choices:
+            return
+        choice = choices[0]
         if not getattr(choice, "finish_reason", None):
             return
         try:
diff --git a/releasenotes/notes/fix-token-extraction-0133808742374ef4.yaml b/releasenotes/notes/fix-token-extraction-0133808742374ef4.yaml
new file mode 100644
index 00000000000..cc8c1aa127b
--- /dev/null
+++ b/releasenotes/notes/fix-token-extraction-0133808742374ef4.yaml
@@ -0,0 +1,4 @@
+---
+fixes:
+  - |
+    LLM Observability: This fix resolves an issue where extracting token metadata from openai streamed chat completion token chunks caused an IndexError.