From bee1817819ba76c5ff9cf299b4e2e74191120470 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Wed, 8 Jan 2025 17:02:03 +0100 Subject: [PATCH 01/25] draft initial implementation of Realtime API --- python/pyproject.toml | 6 + .../audio/04-chat_with_realtime_api.py | 126 +++++ python/samples/concepts/audio/audio_player.py | 2 +- .../concepts/audio/audio_player_async.py | 75 +++ .../concepts/audio/audio_recorder_stream.py | 59 +++ .../ai/chat_completion_client_base.py | 28 +- .../open_ai_realtime_execution_settings.py | 48 ++ .../ai/open_ai/services/open_ai_realtime.py | 66 +++ .../open_ai/services/open_ai_realtime_base.py | 294 +++++++++++ .../services/open_ai_realtime_utils.py | 47 ++ .../connectors/ai/realtime_client_base.py | 51 ++ .../contents/chat_message_content.py | 2 + .../contents/function_call_content.py | 1 + .../streaming_chat_message_content.py | 2 + .../tests/unit/contents/test_audio_content.py | 60 +++ python/uv.lock | 464 ++++++++++-------- 16 files changed, 1120 insertions(+), 211 deletions(-) create mode 100644 python/samples/concepts/audio/04-chat_with_realtime_api.py create mode 100644 python/samples/concepts/audio/audio_player_async.py create mode 100644 python/samples/concepts/audio/audio_recorder_stream.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_utils.py create mode 100644 python/semantic_kernel/connectors/ai/realtime_client_base.py create mode 100644 python/tests/unit/contents/test_audio_content.py diff --git a/python/pyproject.toml b/python/pyproject.toml index 0dc38b0b57f9..d6c7b2d42673 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -123,6 +123,12 @@ dapr = [ "dapr-ext-fastapi>=1.14.0", "flask-dapr>=1.14.0" ] +openai_realtime = [ + "openai[realtime] ~= 1.0", + "pyaudio", + "pydub", + "sounddevice" +] [tool.uv] prerelease = "if-necessary-or-explicit" diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py new file mode 100644 index 000000000000..4440d13b8eec --- /dev/null +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -0,0 +1,126 @@ +# Copyright (c) Microsoft. All rights reserved. +import asyncio +import contextlib +import logging +import signal + +from samples.concepts.audio.audio_player_async import AudioPlayerAsync + +# This simple sample demonstrates how to use the OpenAI Realtime API to create +# a chat bot that can listen and respond directly through audio. +# It requires installing semantic-kernel[openai_realtime] which includes the +# OpenAI Realtime API client and some packages for handling audio locally. +# It has hardcoded device id's set in the AudioRecorderStream and AudioPlayerAsync classes, +# so you may need to adjust these for your system. +from samples.concepts.audio.audio_recorder_stream import AudioRecorderStream +from semantic_kernel import Kernel +from semantic_kernel.connectors.ai import FunctionChoiceBehavior +from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( + OpenAIRealtimeExecutionSettings, + TurnDetection, +) +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime +from semantic_kernel.contents import AudioContent, ChatHistory, StreamingTextContent +from semantic_kernel.functions import kernel_function + +logging.basicConfig(level=logging.WARNING) +logger = logging.getLogger(__name__) + + +def signal_handler(): + for task in asyncio.all_tasks(): + task.cancel() + + +system_message = """ +You are a chat bot. Your name is Mosscap and +you have one goal: figure out what people need. +Your full name, should you need to know it, is +Splendid Speckled Mosscap. You communicate +effectively, but you tend to answer with long +flowery prose. +""" + +history = ChatHistory() +history.add_user_message("Hi there, who are you?") +history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") + + +class Speaker: + def __init__(self, audio_player: AudioPlayerAsync, realtime_client: OpenAIRealtime, kernel: Kernel): + self.audio_player = audio_player + self.realtime_client = realtime_client + self.kernel = kernel + + async def play( + self, + chat_history: ChatHistory, + settings: OpenAIRealtimeExecutionSettings, + ) -> None: + self.audio_player.reset_frame_count() + print("Mosscap (transcript): ", end="") + try: + async for content in self.realtime_client.get_streaming_chat_message_content( + chat_history=chat_history, settings=settings, kernel=self.kernel + ): + if not content: + continue + for item in content.items: + match item: + case StreamingTextContent(): + print(item.text, end="") + await asyncio.sleep(0.01) + continue + case AudioContent(): + self.audio_player.add_data(item.data) + await asyncio.sleep(0.01) + continue + except asyncio.CancelledError: + print("\nThanks for talking to Mosscap!") + + +class Microphone: + def __init__(self, audio_recorder: AudioRecorderStream, realtime_client: OpenAIRealtime): + self.audio_recorder = audio_recorder + self.realtime_client = realtime_client + + async def record_audio(self): + with contextlib.suppress(asyncio.CancelledError): + async for audio in self.audio_recorder.stream_audio_content(): + if audio.data: + await self.realtime_client.send_content(content=audio) + await asyncio.sleep(0.01) + + +@kernel_function +def get_weather(location: str) -> str: + """Get the weather for a location.""" + logger.debug(f"Getting weather for {location}") + return f"The weather in {location} is sunny." + + +async def main() -> None: + loop = asyncio.get_event_loop() + loop.add_signal_handler(signal.SIGINT, signal_handler) + settings = OpenAIRealtimeExecutionSettings( + instructions=system_message, + voice="sage", + turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), + function_choice_behavior=FunctionChoiceBehavior.Auto(), + ) + realtime_client = OpenAIRealtime(ai_model_id="gpt-4o-realtime-preview-2024-12-17") + kernel = Kernel() + kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) + + speaker = Speaker(AudioPlayerAsync(), realtime_client, kernel) + microphone = Microphone(AudioRecorderStream(), realtime_client) + with contextlib.suppress(asyncio.CancelledError): + await asyncio.gather(*[speaker.play(history, settings), microphone.record_audio()]) + + +if __name__ == "__main__": + print( + "Instruction: start speaking, when you stop the API should detect you finished and start responding." + "Press ctrl + c to stop the program." + ) + asyncio.run(main()) diff --git a/python/samples/concepts/audio/audio_player.py b/python/samples/concepts/audio/audio_player.py index b10c15184821..036b978dcff1 100644 --- a/python/samples/concepts/audio/audio_player.py +++ b/python/samples/concepts/audio/audio_player.py @@ -20,7 +20,7 @@ class AudioPlayer(BaseModel): # Audio replay parameters CHUNK: ClassVar[int] = 1024 - audio_content: AudioContent + audio_content: AudioContent | None = None def play(self, text: str | None = None) -> None: """Play the audio content to the default audio output device. diff --git a/python/samples/concepts/audio/audio_player_async.py b/python/samples/concepts/audio/audio_player_async.py new file mode 100644 index 000000000000..9ae424b01c66 --- /dev/null +++ b/python/samples/concepts/audio/audio_player_async.py @@ -0,0 +1,75 @@ +# Copyright (c) Microsoft. All rights reserved. + +import threading + +import numpy as np +import pyaudio +import sounddevice as sd + +CHUNK_LENGTH_S = 0.05 # 100ms +SAMPLE_RATE = 24000 +FORMAT = pyaudio.paInt16 +CHANNELS = 1 + + +class AudioPlayerAsync: + def __init__(self): + self.queue = [] + self.lock = threading.Lock() + self.stream = sd.OutputStream( + callback=self.callback, + samplerate=SAMPLE_RATE, + channels=CHANNELS, + dtype=np.int16, + blocksize=int(CHUNK_LENGTH_S * SAMPLE_RATE), + device=3, + ) + self.playing = False + self._frame_count = 0 + + def callback(self, outdata, frames, time, status): # noqa + with self.lock: + data = np.empty(0, dtype=np.int16) + + # get next item from queue if there is still space in the buffer + while len(data) < frames and len(self.queue) > 0: + item = self.queue.pop(0) + frames_needed = frames - len(data) + data = np.concatenate((data, item[:frames_needed])) + if len(item) > frames_needed: + self.queue.insert(0, item[frames_needed:]) + + self._frame_count += len(data) + + # fill the rest of the frames with zeros if there is no more data + if len(data) < frames: + data = np.concatenate((data, np.zeros(frames - len(data), dtype=np.int16))) + + outdata[:] = data.reshape(-1, 1) + + def reset_frame_count(self): + self._frame_count = 0 + + def get_frame_count(self): + return self._frame_count + + def add_data(self, data: bytes): + with self.lock: + # bytes is pcm16 single channel audio data, convert to numpy array + np_data = np.frombuffer(data, dtype=np.int16) + self.queue.append(np_data) + if not self.playing: + self.start() + + def start(self): + self.playing = True + self.stream.start() + + def stop(self): + self.playing = False + self.stream.stop() + with self.lock: + self.queue = [] + + def terminate(self): + self.stream.close() diff --git a/python/samples/concepts/audio/audio_recorder_stream.py b/python/samples/concepts/audio/audio_recorder_stream.py new file mode 100644 index 000000000000..99ac1a9f8141 --- /dev/null +++ b/python/samples/concepts/audio/audio_recorder_stream.py @@ -0,0 +1,59 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import base64 +from collections.abc import AsyncGenerator +from typing import Any, ClassVar, cast + +from pydantic import BaseModel + +from semantic_kernel.contents.audio_content import AudioContent + + +class AudioRecorderStream(BaseModel): + """A class to record audio from the microphone and save it to a WAV file. + + To start recording, press the spacebar. To stop recording, release the spacebar. + + To use as a context manager, that automatically removes the output file after exiting the context: + ``` + with AudioRecorder(output_filepath="output.wav") as recorder: + recorder.start_recording() + # Do something with the recorded audio + ... + ``` + """ + + # Audio recording parameters + CHANNELS: ClassVar[int] = 1 + SAMPLE_RATE: ClassVar[int] = 24000 + CHUNK_LENGTH_S: ClassVar[float] = 0.05 + + async def stream_audio_content(self) -> AsyncGenerator[AudioContent, None]: + import sounddevice as sd # type: ignore + + # device_info = sd.query_devices() + # print(device_info) + + read_size = int(self.SAMPLE_RATE * 0.02) + + stream = sd.InputStream( + channels=self.CHANNELS, + samplerate=self.SAMPLE_RATE, + dtype="int16", + device=4, + ) + stream.start() + try: + while True: + if stream.read_available < read_size: + await asyncio.sleep(0) + continue + + data, _ = stream.read(read_size) + yield AudioContent(data=base64.b64encode(cast(Any, data)), data_format="base64", mime_type="audio/wav") + except KeyboardInterrupt: + pass + finally: + stream.stop() + stream.close() diff --git a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py index 5c527e994564..e24ae4954c42 100644 --- a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py @@ -221,7 +221,7 @@ async def get_streaming_chat_message_contents( if not self.SUPPORTS_FUNCTION_CALLING: async for streaming_chat_message_contents in self._inner_get_streaming_chat_message_contents( - chat_history, settings + chat_history, settings, **kwargs ): yield streaming_chat_message_contents return @@ -246,7 +246,7 @@ async def get_streaming_chat_message_contents( or not settings.function_choice_behavior.auto_invoke_kernel_functions ): async for streaming_chat_message_contents in self._inner_get_streaming_chat_message_contents( - chat_history, settings + chat_history, settings, **kwargs ): yield streaming_chat_message_contents return @@ -258,12 +258,14 @@ async def get_streaming_chat_message_contents( all_messages: list["StreamingChatMessageContent"] = [] function_call_returned = False async for messages in self._inner_get_streaming_chat_message_contents( - chat_history, settings, request_index + chat_history, settings, request_index, **kwargs ): for msg in messages: if msg is not None: all_messages.append(msg) - if any(isinstance(item, FunctionCallContent) for item in msg.items): + if not function_call_returned and any( + isinstance(item, FunctionCallContent) for item in msg.items + ): function_call_returned = True yield messages @@ -307,6 +309,7 @@ async def get_streaming_chat_message_contents( function_invoke_attempt=request_index, ) if self._yield_function_result_messages(function_result_messages): + await self._streaming_function_call_result_callback(function_result_messages) yield function_result_messages if any(result.terminate for result in results if result is not None): @@ -429,7 +432,22 @@ def _get_ai_model_id(self, settings: "PromptExecutionSettings") -> str: return getattr(settings, "ai_model_id", self.ai_model_id) or self.ai_model_id def _yield_function_result_messages(self, function_result_messages: list) -> bool: - """Determine if the function result messages should be yielded.""" + """Determine if the function result messages should be yielded. + + If there are messages and if the first message has items, then yield the messages. + """ return len(function_result_messages) > 0 and len(function_result_messages[0].items) > 0 + async def _streaming_function_call_result_callback( + self, function_result_messages: list["ChatMessageContent"] + ) -> None: + """Callback to handle the streaming function call result messages. + + Override this method to handle the streaming function call result messages. + + Args: + function_result_messages (list): The streaming function call result messages. + """ + return + # endregion diff --git a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py new file mode 100644 index 000000000000..480e2ed1373f --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py @@ -0,0 +1,48 @@ +# Copyright (c) Microsoft. All rights reserved. + +from collections.abc import Sequence +from typing import Annotated, Any, Literal + +from pydantic import Field + +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.kernel_pydantic import KernelBaseModel + + +class TurnDetection(KernelBaseModel): + """Turn detection settings.""" + + type: Literal["server_vad"] | None = None + threshold: Annotated[float | None, Field(ge=0, le=1)] = None + prefix_padding_ms: Annotated[int | None, Field(ge=0)] = None + silence_duration_ms: Annotated[int | None, Field(ge=0)] = None + create_response: bool | None = None + + +class OpenAIRealtimeExecutionSettings(PromptExecutionSettings): + """Request settings for OpenAI realtime services.""" + + modalities: Sequence[Literal["audio", "text"]] | None = None + ai_model_id: Annotated[str | None, Field(None, serialization_alias="model")] = None + instructions: str | None = None + voice: str | None = None + input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | None = None + output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | None = None + input_audio_transcription: dict[str, Any] | None = None + turn_detection: TurnDetection | None = None + tools: Annotated[ + list[dict[str, Any]] | None, + Field( + description="Do not set this manually. It is set by the service based " + "on the function choice configuration.", + ), + ] = None + tool_choice: Annotated[ + str | None, + Field( + description="Do not set this manually. It is set by the service based " + "on the function choice configuration.", + ), + ] = None + temperature: Annotated[float | None, Field(ge=0.0, le=2.0)] = None + max_response_output_tokens: Annotated[int | Literal["inf"] | None, Field(gt=0)] = None diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py new file mode 100644 index 000000000000..23351d7b6176 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -0,0 +1,66 @@ +# Copyright (c) Microsoft. All rights reserved. + +from collections.abc import Mapping + +from openai import AsyncOpenAI +from pydantic import ValidationError + +from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase +from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_base import OpenAIRealtimeBase +from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings +from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError + + +class OpenAIRealtime(OpenAIRealtimeBase, OpenAIConfigBase): + """OpenAI Realtime service.""" + + def __init__( + self, + ai_model_id: str | None = None, + api_key: str | None = None, + org_id: str | None = None, + service_id: str | None = None, + default_headers: Mapping[str, str] | None = None, + async_client: AsyncOpenAI | None = None, + env_file_path: str | None = None, + env_file_encoding: str | None = None, + ) -> None: + """Initialize an OpenAITextCompletion service. + + Args: + ai_model_id (str | None): OpenAI model name, see + https://platform.openai.com/docs/models + service_id (str | None): Service ID tied to the execution settings. + api_key (str | None): The optional API key to use. If provided will override, + the env vars or .env file value. + org_id (str | None): The optional org ID to use. If provided will override, + the env vars or .env file value. + default_headers: The default headers mapping of string keys to + string values for HTTP requests. (Optional) + async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional) + env_file_path (str | None): Use the environment settings file as a fallback to + environment variables. (Optional) + env_file_encoding (str | None): The encoding of the environment settings file. (Optional) + """ + try: + openai_settings = OpenAISettings.create( + api_key=api_key, + org_id=org_id, + text_model_id=ai_model_id, + env_file_path=env_file_path, + env_file_encoding=env_file_encoding, + ) + except ValidationError as ex: + raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex + if not openai_settings.text_model_id: + raise ServiceInitializationError("The OpenAI text model ID is required.") + super().__init__( + ai_model_id=openai_settings.text_model_id, + service_id=service_id, + api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None, + org_id=openai_settings.org_id, + ai_model_type=OpenAIModelTypes.TEXT, + default_headers=default_headers, + client=async_client, + ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py new file mode 100644 index 000000000000..c73f12d7f343 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -0,0 +1,294 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import base64 +import logging +import sys +from collections.abc import AsyncGenerator, Callable +from typing import TYPE_CHECKING, Any, ClassVar + +if sys.version_info >= (3, 12): + from typing import override # pragma: no cover +else: + from typing_extensions import override # pragma: no cover + +from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection +from openai.types.beta.realtime.conversation_item_create_event_param import ConversationItemParam +from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent +from openai.types.beta.realtime.session import Session +from pydantic import Field + +from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase +from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType +from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_utils import ( + update_settings_from_function_call_configuration, +) +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.contents.streaming_text_content import StreamingTextContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel + +if TYPE_CHECKING: + from semantic_kernel.contents.chat_history import ChatHistory + +logger: logging.Logger = logging.getLogger(__name__) + + +class OpenAIRealtimeBase(OpenAIHandler, ChatCompletionClientBase): + """OpenAI Realtime service.""" + + SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True + connection: AsyncRealtimeConnection | None = None + connected: asyncio.Event = Field(default_factory=asyncio.Event) + session: Session | None = None + + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: + """Get the request settings class.""" + from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa + OpenAIRealtimeExecutionSettings, + ) + + return OpenAIRealtimeExecutionSettings + + async def _get_connection(self) -> AsyncRealtimeConnection: + await self.connected.wait() + if not self.connection: + raise ValueError("Connection not established") + return self.connection + + @override + async def _inner_get_streaming_chat_message_contents( + self, + chat_history: "ChatHistory", + settings: "PromptExecutionSettings", + function_invoke_attempt: int = 0, + **kwargs: Any, + ) -> AsyncGenerator[list[StreamingChatMessageContent], Any]: + if not isinstance(settings, self.get_prompt_execution_settings_class()): + settings = self.get_prompt_execution_settings_from_settings(settings) + + events: list[RealtimeServerEvent] = [] + detailed_events: dict[str, list[RealtimeServerEvent]] = {} + function_calls: list[StreamingChatMessageContent] = [] + + async with self.client.beta.realtime.connect(model=self.ai_model_id) as conn: + self.connection = conn + self.connected.set() + + await conn.session.update(session=settings.prepare_settings_dict()) + if len(chat_history) > 0: + await asyncio.gather(*(self._add_content_to_conversation(msg) for msg in chat_history.messages)) + + async for event in conn: + events.append(event) + detailed_events.setdefault(event.type, []).append(event) + match event.type: + case "session.created" | "session.updated": + self.session = event.session + continue + case "error": + logger.error("Error received: %s", event.error) + continue + case "response.audio.delta": + yield [ + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[AudioContent(data=base64.b64decode(event.delta), data_format="base64")], + choice_index=event.content_index, + inner_content=event, + ) + ] + continue + case "response.audio_transcript.delta": + yield [ + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[StreamingTextContent(text=event.delta, choice_index=event.content_index)], + choice_index=event.content_index, + inner_content=event, + ) + ] + continue + case "response.audio_transcript.done": + chat_history.add_message( + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[StreamingTextContent(text=event.transcript, choice_index=event.content_index)], + choice_index=event.content_index, + inner_content=event, + ) + ) + case "response.function_call_arguments.delta": + msg = StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + ], + choice_index=0, + inner_content=event, + ) + function_calls.append(msg) + yield [msg] + continue + case "response.function_call_arguments.done": + # execute function, add result to conversation + if len(function_calls) > 0: + function_call = sum(function_calls[1:], function_calls[0]) + # execute function + results = [] + for item in function_call.items: + if isinstance(item, FunctionCallContent): + kernel: Kernel | None = kwargs.get("kernel") + call_id = item.name + function_name = next( + output_item_event.item.name + for output_item_event in detailed_events["response.output_item.added"] + if output_item_event.item.call_id == call_id + ) + item.plugin_name, item.function_name = function_name.split("-", 1) + if kernel: + await kernel.invoke_function_call(item, chat_history) + # add result to conversation + results.append(chat_history.messages[-1]) + for message in results: + await self._add_content_to_conversation(content=message) + case _: + logger.debug("Unhandled event type: %s", event.type) + logger.debug(f"Finished streaming chat message contents, {len(events)} events received.") + for event_type in detailed_events: + logger.debug(f"Event type: {event_type}, count: {len(detailed_events[event_type])}") + + async def send_content( + self, + content: ChatMessageContent | AudioContent | AsyncGenerator[AudioContent, Any], + **kwargs: Any, + ) -> None: + """Send a chat message content to the service. + + This content should contain audio content, either as a ChatMessageContent with a + AudioContent item, as AudioContent directly, as or as a generator of AudioContent. + + """ + if isinstance(content, AudioContent | ChatMessageContent): + if isinstance(content, ChatMessageContent): + content = next(item for item in content.items if isinstance(item, AudioContent)) + connection = await self._get_connection() + await connection.input_audio_buffer.append(audio=content.data.decode("utf-8")) + await asyncio.sleep(0) + return + + async for audio_content in content: + if isinstance(audio_content, ChatMessageContent): + audio_content = next(item for item in audio_content.items if isinstance(item, AudioContent)) + connection = await self._get_connection() + await connection.input_audio_buffer.append(audio=audio_content.data.decode("utf-8")) + await asyncio.sleep(0) + + async def commit_content(self, settings: "PromptExecutionSettings") -> None: + """Commit the chat message content to the service. + + This is only needed when turn detection is not handled by the service. + + This behavior is determined by the turn_detection parameter in the settings. + If turn_detection is None, then it will commit the audio buffer and + ask the service to process the audio and create the response. + """ + if not isinstance(settings, self.get_prompt_execution_settings_class()): + settings = self.get_prompt_execution_settings_from_settings(settings) + if not settings.turn_detection: + connection = await self._get_connection() + await connection.input_audio_buffer.commit() + await connection.response.create() + + @override + def _update_function_choice_settings_callback( + self, + ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: + return update_settings_from_function_call_configuration + + async def _streaming_function_call_result_callback( + self, function_result_messages: list[StreamingChatMessageContent] + ) -> None: + """Callback to handle the streaming function call result messages. + + Override this method to handle the streaming function call result messages. + + Args: + function_result_messages (list): The streaming function call result messages. + """ + for msg in function_result_messages: + await self._add_content_to_conversation(msg) + + async def _add_content_to_conversation(self, content: ChatMessageContent) -> None: + """Add an item to the conversation.""" + connection = await self._get_connection() + for item in content.items: + match item: + case AudioContent(): + await connection.conversation.item.create( + item=ConversationItemParam( + type="message", + content=[ + { + "type": "input_audio", + "audio": item.data.decode("utf-8"), + } + ], + role="user", + ) + ) + case TextContent(): + await connection.conversation.item.create( + item=ConversationItemParam( + type="message", + content=[ + { + "type": "input_text", + "text": item.text, + } + ], + role="user", + ) + ) + case FunctionCallContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function call needs to have a call_id") + continue + await connection.conversation.item.create( + item=ConversationItemParam( + type="function_call", + name=item.name, + arguments=item.arguments, + call_id=call_id, + ) + ) + case FunctionResultContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function result needs to have a call_id") + continue + await connection.conversation.item.create( + item=ConversationItemParam( + type="function_call_output", + output=item.result, + call_id=call_id, + ) + ) + case _: + logger.debug("Unhandled item type: %s", item.__class__.__name__) + continue diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_utils.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_utils.py new file mode 100644 index 000000000000..ada8d42924c0 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_utils.py @@ -0,0 +1,47 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from semantic_kernel.connectors.ai.function_choice_behavior import ( + FunctionCallChoiceConfiguration, + FunctionChoiceType, + ) + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings + from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata + + +def update_settings_from_function_call_configuration( + function_choice_configuration: "FunctionCallChoiceConfiguration", + settings: "PromptExecutionSettings", + type: "FunctionChoiceType", +) -> None: + """Update the settings from a FunctionChoiceConfiguration.""" + if ( + function_choice_configuration.available_functions + and hasattr(settings, "tool_choice") + and hasattr(settings, "tools") + ): + settings.tool_choice = type + settings.tools = [ + kernel_function_metadata_to_function_call_format(f) + for f in function_choice_configuration.available_functions + ] + + +def kernel_function_metadata_to_function_call_format( + metadata: "KernelFunctionMetadata", +) -> dict[str, Any]: + """Convert the kernel function metadata to function calling format.""" + return { + "type": "function", + "name": metadata.fully_qualified_name, + "description": metadata.description or "", + "parameters": { + "type": "object", + "properties": { + param.name: param.schema_data for param in metadata.parameters if param.include_in_function_choices + }, + "required": [p.name for p in metadata.parameters if p.is_required and p.include_in_function_choices], + }, + } diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py new file mode 100644 index 000000000000..734e7e7caed4 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -0,0 +1,51 @@ +# Copyright (c) Microsoft. All rights reserved. + + +from abc import ABC, abstractmethod +from collections.abc import AsyncGenerator +from typing import Any + +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.services.ai_service_client_base import AIServiceClientBase + + +class RealtimeClientBase(AIServiceClientBase, ABC): + """Base class for audio to text client.""" + + @abstractmethod + async def receive( + self, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> AsyncGenerator[TextContent | AudioContent, Any]: + """Get text contents from audio. + + Args: + settings: Prompt execution settings. + kwargs: Additional arguments. + + Returns: + list[TextContent | AudioContent]: response contents. + """ + raise NotImplementedError + + @abstractmethod + async def send( + self, + audio_content: AudioContent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> None: + """Get text content from audio. + + Args: + audio_content: Audio content. + settings: Prompt execution settings. + kwargs: Additional arguments. + + Returns: + TextContent: Text content. + """ + raise NotImplementedError diff --git a/python/semantic_kernel/contents/chat_message_content.py b/python/semantic_kernel/contents/chat_message_content.py index ce52a7428831..42e50013976c 100644 --- a/python/semantic_kernel/contents/chat_message_content.py +++ b/python/semantic_kernel/contents/chat_message_content.py @@ -10,6 +10,7 @@ from pydantic import Field from semantic_kernel.contents.annotation_content import AnnotationContent +from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.const import ( ANNOTATION_CONTENT_TAG, CHAT_MESSAGE_CONTENT_TAG, @@ -56,6 +57,7 @@ | FileReferenceContent | StreamingAnnotationContent | StreamingFileReferenceContent + | AudioContent ) logger = logging.getLogger(__name__) diff --git a/python/semantic_kernel/contents/function_call_content.py b/python/semantic_kernel/contents/function_call_content.py index 7067311f4c8a..a8b2509336e1 100644 --- a/python/semantic_kernel/contents/function_call_content.py +++ b/python/semantic_kernel/contents/function_call_content.py @@ -124,6 +124,7 @@ def __add__(self, other: "FunctionCallContent | None") -> "FunctionCallContent": index=self.index or other.index, name=self.name or other.name, arguments=self.combine_arguments(self.arguments, other.arguments), + metadata=self.metadata | other.metadata, ) def combine_arguments( diff --git a/python/semantic_kernel/contents/streaming_chat_message_content.py b/python/semantic_kernel/contents/streaming_chat_message_content.py index be1ca56f113b..926d140ed241 100644 --- a/python/semantic_kernel/contents/streaming_chat_message_content.py +++ b/python/semantic_kernel/contents/streaming_chat_message_content.py @@ -6,6 +6,7 @@ from pydantic import Field +from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.chat_message_content import ChatMessageContent from semantic_kernel.contents.function_call_content import FunctionCallContent from semantic_kernel.contents.function_result_content import FunctionResultContent @@ -20,6 +21,7 @@ from semantic_kernel.exceptions import ContentAdditionException ITEM_TYPES = Union[ + AudioContent, ImageContent, StreamingTextContent, FunctionCallContent, diff --git a/python/tests/unit/contents/test_audio_content.py b/python/tests/unit/contents/test_audio_content.py new file mode 100644 index 000000000000..2af5a99b9e29 --- /dev/null +++ b/python/tests/unit/contents/test_audio_content.py @@ -0,0 +1,60 @@ +# Copyright (c) Microsoft. All rights reserved. + +import os + +import pytest + +from semantic_kernel.contents.audio_content import AudioContent + +test_cases = [ + pytest.param(AudioContent(uri="http://test_uri"), id="uri"), + pytest.param(AudioContent(data=b"test_data", mime_type="image/jpeg", data_format="base64"), id="data"), + pytest.param(AudioContent(uri="http://test_uri", data=b"test_data", mime_type="image/jpeg"), id="both"), + pytest.param( + AudioContent.from_image_path( + image_path=os.path.join(os.path.dirname(__file__), "../../", "assets/sample_image.jpg") + ), + id="image_file", + ), +] + + +def test_create_uri(): + image = AudioContent(uri="http://test_uri") + assert str(image.uri) == "http://test_uri/" + + +def test_create_file_from_path(): + image_path = os.path.join(os.path.dirname(__file__), "../../", "assets/sample_image.jpg") + image = AudioContent.from_image_path(image_path=image_path) + assert image.mime_type == "image/jpeg" + assert image.data_uri.startswith("data:image/jpeg;") + assert image.data is not None + + +def test_create_data(): + image = AudioContent(data=b"test_data", mime_type="image/jpeg") + assert image.mime_type == "image/jpeg" + assert image.data == b"test_data" + + +def test_to_str_uri(): + image = AudioContent(uri="http://test_uri") + assert str(image) == "http://test_uri/" + + +def test_to_str_data(): + image = AudioContent(data=b"test_data", mime_type="image/jpeg", data_format="base64") + assert str(image) == "data:image/jpeg;base64,dGVzdF9kYXRh" + + +@pytest.mark.parametrize("image", test_cases) +def test_element_roundtrip(image): + element = image.to_element() + new_image = AudioContent.from_element(element) + assert new_image == image + + +@pytest.mark.parametrize("image", test_cases) +def test_to_dict(image): + assert image.to_dict() == {"type": "image_url", "image_url": {"url": str(image)}} diff --git a/python/uv.lock b/python/uv.lock index 6daaa60a388a..301c71febcfb 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -148,7 +148,7 @@ wheels = [ [[package]] name = "anthropic" -version = "0.43.0" +version = "0.42.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -159,9 +159,9 @@ dependencies = [ { name = "sniffio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e1/0a/973d2ac6c9f7d1be41829c7b878cbe399385b25cc2ebe80ad0eec9999b8c/anthropic-0.43.0.tar.gz", hash = "sha256:06801f01d317a431d883230024318d48981758058bf7e079f33fb11f64b5a5c1", size = 194826 } +sdist = { url = "https://files.pythonhosted.org/packages/e7/7c/91b79f5ae4a52497a4e330d66ea5929aec2878ee2c9f8a998dbe4f4c7f01/anthropic-0.42.0.tar.gz", hash = "sha256:bf8b0ed8c8cb2c2118038f29c58099d2f99f7847296cafdaa853910bfff4edf4", size = 192361 } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/88/ded3ba979a2218a448cbc1a1e762d998b92f30529452c5104b35b6cb71f8/anthropic-0.43.0-py3-none-any.whl", hash = "sha256:f748a703f77b3244975e1aace3a935840dc653a4714fb6bba644f97cc76847b4", size = 207867 }, + { url = "https://files.pythonhosted.org/packages/ba/33/b907a6d27dd0d8d3adb4edb5c9e9c85a189719ec6855051cce3814c8ef13/anthropic-0.42.0-py3-none-any.whl", hash = "sha256:46775f65b723c078a2ac9e9de44a46db5c6a4fabeacfd165e5ea78e6817f4eff", size = 203365 }, ] [[package]] @@ -414,30 +414,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.36.1" +version = "1.35.92" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "s3transfer", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/04/0c6cea060653eee75f4348152dfc0aa0b241f7d1f99a530079ee44d61e4b/boto3-1.36.1.tar.gz", hash = "sha256:258ab77225a81d3cf3029c9afe9920cd9dec317689dfadec6f6f0a23130bb60a", size = 110959 } +sdist = { url = "https://files.pythonhosted.org/packages/3f/de/a96f2aa9a5770932e5bc3a9d3a6b4e0270487d5846a3387d5f5148e4c974/boto3-1.35.92.tar.gz", hash = "sha256:f7851cb320dcb2a53fc73b4075187ec9b05d51291539601fa238623fdc0e8cd3", size = 111016 } wheels = [ - { url = "https://files.pythonhosted.org/packages/2b/ed/464e1df3901fbfedd5a0786e551240216f0c867440fa6156595178227b3f/boto3-1.36.1-py3-none-any.whl", hash = "sha256:eb21380d73fec6645439c0d802210f72a0cdb3295b02953f246ff53f512faa8f", size = 139163 }, + { url = "https://files.pythonhosted.org/packages/4e/9d/0f7ecfea26ba0524617f7cfbd0b188d963bbc3b4cf2d9c3441dffe310c30/boto3-1.35.92-py3-none-any.whl", hash = "sha256:786930d5f1cd13d03db59ff2abbb2b7ffc173fd66646d5d8bee07f316a5f16ca", size = 139179 }, ] [[package]] name = "botocore" -version = "1.36.1" +version = "1.35.92" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "python-dateutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/aa/556720b3ee9629b7c4366b5a0d9797a84e83a97f78435904cbb9bdc41939/botocore-1.36.1.tar.gz", hash = "sha256:f789a6f272b5b3d8f8756495019785e33868e5e00dd9662a3ee7959ac939bb12", size = 13498150 } +sdist = { url = "https://files.pythonhosted.org/packages/bf/e1/4f3d4e43d10a4070aa43c6d9c0cfd40fe53dbd1c81a31f237c29a86735a3/botocore-1.35.92.tar.gz", hash = "sha256:caa7d5d857fed5b3d694b89c45f82b9f938f840e90a4eb7bf50aa65da2ba8f82", size = 13494438 } wheels = [ - { url = "https://files.pythonhosted.org/packages/be/bb/5431f12e2dadd881fd023fb57e7e3ab82f7b697c38dc837fc8d70cca51bd/botocore-1.36.1-py3-none-any.whl", hash = "sha256:dec513b4eb8a847d79bbefdcdd07040ed9d44c20b0001136f0890a03d595705a", size = 13297686 }, + { url = "https://files.pythonhosted.org/packages/a6/6f/015482b4bb28e9edcde97b67ec2d40f84956e1b8c7b22254f58a461d357d/botocore-1.35.92-py3-none-any.whl", hash = "sha256:f94ae1e056a675bd67c8af98a6858d06e3927d974d6c712ed6e27bb1d11bee1d", size = 13300322 }, ] [[package]] @@ -874,27 +874,27 @@ wheels = [ [[package]] name = "debugpy" -version = "1.8.12" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/68/25/c74e337134edf55c4dfc9af579eccb45af2393c40960e2795a94351e8140/debugpy-1.8.12.tar.gz", hash = "sha256:646530b04f45c830ceae8e491ca1c9320a2d2f0efea3141487c82130aba70dce", size = 1641122 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/56/19/dd58334c0a1ec07babf80bf29fb8daf1a7ca4c1a3bbe61548e40616ac087/debugpy-1.8.12-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:a2ba7ffe58efeae5b8fad1165357edfe01464f9aef25e814e891ec690e7dd82a", size = 2076091 }, - { url = "https://files.pythonhosted.org/packages/4c/37/bde1737da15f9617d11ab7b8d5267165f1b7dae116b2585a6643e89e1fa2/debugpy-1.8.12-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbbd4149c4fc5e7d508ece083e78c17442ee13b0e69bfa6bd63003e486770f45", size = 3560717 }, - { url = "https://files.pythonhosted.org/packages/d9/ca/bc67f5a36a7de072908bc9e1156c0f0b272a9a2224cf21540ab1ffd71a1f/debugpy-1.8.12-cp310-cp310-win32.whl", hash = "sha256:b202f591204023b3ce62ff9a47baa555dc00bb092219abf5caf0e3718ac20e7c", size = 5180672 }, - { url = "https://files.pythonhosted.org/packages/c1/b9/e899c0a80dfa674dbc992f36f2b1453cd1ee879143cdb455bc04fce999da/debugpy-1.8.12-cp310-cp310-win_amd64.whl", hash = "sha256:9649eced17a98ce816756ce50433b2dd85dfa7bc92ceb60579d68c053f98dff9", size = 5212702 }, - { url = "https://files.pythonhosted.org/packages/af/9f/5b8af282253615296264d4ef62d14a8686f0dcdebb31a669374e22fff0a4/debugpy-1.8.12-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:36f4829839ef0afdfdd208bb54f4c3d0eea86106d719811681a8627ae2e53dd5", size = 2174643 }, - { url = "https://files.pythonhosted.org/packages/ef/31/f9274dcd3b0f9f7d1e60373c3fa4696a585c55acb30729d313bb9d3bcbd1/debugpy-1.8.12-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a28ed481d530e3138553be60991d2d61103ce6da254e51547b79549675f539b7", size = 3133457 }, - { url = "https://files.pythonhosted.org/packages/ab/ca/6ee59e9892e424477e0c76e3798046f1fd1288040b927319c7a7b0baa484/debugpy-1.8.12-cp311-cp311-win32.whl", hash = "sha256:4ad9a94d8f5c9b954e0e3b137cc64ef3f579d0df3c3698fe9c3734ee397e4abb", size = 5106220 }, - { url = "https://files.pythonhosted.org/packages/d5/1a/8ab508ab05ede8a4eae3b139bbc06ea3ca6234f9e8c02713a044f253be5e/debugpy-1.8.12-cp311-cp311-win_amd64.whl", hash = "sha256:4703575b78dd697b294f8c65588dc86874ed787b7348c65da70cfc885efdf1e1", size = 5130481 }, - { url = "https://files.pythonhosted.org/packages/ba/e6/0f876ecfe5831ebe4762b19214364753c8bc2b357d28c5d739a1e88325c7/debugpy-1.8.12-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:7e94b643b19e8feb5215fa508aee531387494bf668b2eca27fa769ea11d9f498", size = 2500846 }, - { url = "https://files.pythonhosted.org/packages/19/64/33f41653a701f3cd2cbff8b41ebaad59885b3428b5afd0d93d16012ecf17/debugpy-1.8.12-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:086b32e233e89a2740c1615c2f775c34ae951508b28b308681dbbb87bba97d06", size = 4222181 }, - { url = "https://files.pythonhosted.org/packages/32/a6/02646cfe50bfacc9b71321c47dc19a46e35f4e0aceea227b6d205e900e34/debugpy-1.8.12-cp312-cp312-win32.whl", hash = "sha256:2ae5df899732a6051b49ea2632a9ea67f929604fd2b036613a9f12bc3163b92d", size = 5227017 }, - { url = "https://files.pythonhosted.org/packages/da/a6/10056431b5c47103474312cf4a2ec1001f73e0b63b1216706d5fef2531eb/debugpy-1.8.12-cp312-cp312-win_amd64.whl", hash = "sha256:39dfbb6fa09f12fae32639e3286112fc35ae976114f1f3d37375f3130a820969", size = 5267555 }, - { url = "https://files.pythonhosted.org/packages/cf/4d/7c3896619a8791effd5d8c31f0834471fc8f8fb3047ec4f5fc69dd1393dd/debugpy-1.8.12-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:696d8ae4dff4cbd06bf6b10d671e088b66669f110c7c4e18a44c43cf75ce966f", size = 2485246 }, - { url = "https://files.pythonhosted.org/packages/99/46/bc6dcfd7eb8cc969a5716d858e32485eb40c72c6a8dc88d1e3a4d5e95813/debugpy-1.8.12-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:898fba72b81a654e74412a67c7e0a81e89723cfe2a3ea6fcd3feaa3395138ca9", size = 4218616 }, - { url = "https://files.pythonhosted.org/packages/03/dd/d7fcdf0381a9b8094da1f6a1c9f19fed493a4f8576a2682349b3a8b20ec7/debugpy-1.8.12-cp313-cp313-win32.whl", hash = "sha256:22a11c493c70413a01ed03f01c3c3a2fc4478fc6ee186e340487b2edcd6f4180", size = 5226540 }, - { url = "https://files.pythonhosted.org/packages/25/bd/ecb98f5b5fc7ea0bfbb3c355bc1dd57c198a28780beadd1e19915bf7b4d9/debugpy-1.8.12-cp313-cp313-win_amd64.whl", hash = "sha256:fdb3c6d342825ea10b90e43d7f20f01535a72b3a1997850c0c3cefa5c27a4a2c", size = 5267134 }, - { url = "https://files.pythonhosted.org/packages/38/c4/5120ad36405c3008f451f94b8f92ef1805b1e516f6ff870f331ccb3c4cc0/debugpy-1.8.12-py2.py3-none-any.whl", hash = "sha256:274b6a2040349b5c9864e475284bce5bb062e63dce368a394b8cc865ae3b00c6", size = 5229490 }, +version = "1.8.11" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/bc/e7/666f4c9b0e24796af50aadc28d36d21c2e01e831a934535f956e09b3650c/debugpy-1.8.11.tar.gz", hash = "sha256:6ad2688b69235c43b020e04fecccdf6a96c8943ca9c2fb340b8adc103c655e57", size = 1640124 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/26/e6/4cf7422eaa591b4c7d6a9fde224095dac25283fdd99d90164f28714242b0/debugpy-1.8.11-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:2b26fefc4e31ff85593d68b9022e35e8925714a10ab4858fb1b577a8a48cb8cd", size = 2075100 }, + { url = "https://files.pythonhosted.org/packages/83/3a/e163de1df5995d95760a4d748b02fbefb1c1bf19e915b664017c40435dbf/debugpy-1.8.11-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:61bc8b3b265e6949855300e84dc93d02d7a3a637f2aec6d382afd4ceb9120c9f", size = 3559724 }, + { url = "https://files.pythonhosted.org/packages/27/6c/327e19fd1bf428a1efe1a6f97b306689c54c2cebcf871b66674ead718756/debugpy-1.8.11-cp310-cp310-win32.whl", hash = "sha256:c928bbf47f65288574b78518449edaa46c82572d340e2750889bbf8cd92f3737", size = 5178068 }, + { url = "https://files.pythonhosted.org/packages/49/80/359ff8aa388f0bd4a48f0fa9ce3606396d576657ac149c6fba3cc7de8adb/debugpy-1.8.11-cp310-cp310-win_amd64.whl", hash = "sha256:8da1db4ca4f22583e834dcabdc7832e56fe16275253ee53ba66627b86e304da1", size = 5210109 }, + { url = "https://files.pythonhosted.org/packages/7c/58/8e3f7ec86c1b7985a232667b5df8f3b1b1c8401028d8f4d75e025c9556cd/debugpy-1.8.11-cp311-cp311-macosx_14_0_universal2.whl", hash = "sha256:85de8474ad53ad546ff1c7c7c89230db215b9b8a02754d41cb5a76f70d0be296", size = 2173656 }, + { url = "https://files.pythonhosted.org/packages/d2/03/95738a68ade2358e5a4d63a2fd8e7ed9ad911001cfabbbb33a7f81343945/debugpy-1.8.11-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8ffc382e4afa4aee367bf413f55ed17bd91b191dcaf979890af239dda435f2a1", size = 3132464 }, + { url = "https://files.pythonhosted.org/packages/ca/f4/18204891ab67300950615a6ad09b9de236203a9138f52b3b596fa17628ca/debugpy-1.8.11-cp311-cp311-win32.whl", hash = "sha256:40499a9979c55f72f4eb2fc38695419546b62594f8af194b879d2a18439c97a9", size = 5103637 }, + { url = "https://files.pythonhosted.org/packages/3b/90/3775e301cfa573b51eb8a108285681f43f5441dc4c3916feed9f386ef861/debugpy-1.8.11-cp311-cp311-win_amd64.whl", hash = "sha256:987bce16e86efa86f747d5151c54e91b3c1e36acc03ce1ddb50f9d09d16ded0e", size = 5127862 }, + { url = "https://files.pythonhosted.org/packages/c6/ae/2cf26f3111e9d94384d9c01e9d6170188b0aeda15b60a4ac6457f7c8a26f/debugpy-1.8.11-cp312-cp312-macosx_14_0_universal2.whl", hash = "sha256:84e511a7545d11683d32cdb8f809ef63fc17ea2a00455cc62d0a4dbb4ed1c308", size = 2498756 }, + { url = "https://files.pythonhosted.org/packages/b0/16/ec551789d547541a46831a19aa15c147741133da188e7e6acf77510545a7/debugpy-1.8.11-cp312-cp312-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ce291a5aca4985d82875d6779f61375e959208cdf09fcec40001e65fb0a54768", size = 4219136 }, + { url = "https://files.pythonhosted.org/packages/72/6f/b2b3ce673c55f882d27a6eb04a5f0c68bcad6b742ac08a86d8392ae58030/debugpy-1.8.11-cp312-cp312-win32.whl", hash = "sha256:28e45b3f827d3bf2592f3cf7ae63282e859f3259db44ed2b129093ca0ac7940b", size = 5224440 }, + { url = "https://files.pythonhosted.org/packages/77/09/b1f05be802c1caef5b3efc042fc6a7cadd13d8118b072afd04a9b9e91e06/debugpy-1.8.11-cp312-cp312-win_amd64.whl", hash = "sha256:44b1b8e6253bceada11f714acf4309ffb98bfa9ac55e4fce14f9e5d4484287a1", size = 5264578 }, + { url = "https://files.pythonhosted.org/packages/2e/66/931dc2479aa8fbf362dc6dcee707d895a84b0b2d7b64020135f20b8db1ed/debugpy-1.8.11-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:8988f7163e4381b0da7696f37eec7aca19deb02e500245df68a7159739bbd0d3", size = 2483651 }, + { url = "https://files.pythonhosted.org/packages/10/07/6c171d0fe6b8d237e35598b742f20ba062511b3a4631938cc78eefbbf847/debugpy-1.8.11-cp313-cp313-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6c1f6a173d1140e557347419767d2b14ac1c9cd847e0b4c5444c7f3144697e4e", size = 4213770 }, + { url = "https://files.pythonhosted.org/packages/89/f1/0711da6ac250d4fe3bf7b3e9b14b4a86e82a98b7825075c07e19bab8da3d/debugpy-1.8.11-cp313-cp313-win32.whl", hash = "sha256:bb3b15e25891f38da3ca0740271e63ab9db61f41d4d8541745cfc1824252cb28", size = 5223911 }, + { url = "https://files.pythonhosted.org/packages/56/98/5e27fa39050749ed460025bcd0034a0a5e78a580a14079b164cc3abdeb98/debugpy-1.8.11-cp313-cp313-win_amd64.whl", hash = "sha256:d8768edcbeb34da9e11bcb8b5c2e0958d25218df7a6e56adf415ef262cd7b6d1", size = 5264166 }, + { url = "https://files.pythonhosted.org/packages/77/0a/d29a5aacf47b4383ed569b8478c02d59ee3a01ad91224d2cff8562410e43/debugpy-1.8.11-py2.py3-none-any.whl", hash = "sha256:0e22f846f4211383e6a416d04b4c13ed174d24cc5d43f5fd52e7821d0ebc8920", size = 5226874 }, ] [[package]] @@ -1207,7 +1207,7 @@ grpc = [ [[package]] name = "google-api-python-client" -version = "2.159.0" +version = "2.157.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -1216,9 +1216,9 @@ dependencies = [ { name = "httplib2", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "uritemplate", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/9f/12b58cca5a93d63fd6a7abed570423bdf2db4349eb9361ac5214d42ed7d6/google_api_python_client-2.159.0.tar.gz", hash = "sha256:55197f430f25c907394b44fa078545ffef89d33fd4dca501b7db9f0d8e224bd6", size = 12302576 } +sdist = { url = "https://files.pythonhosted.org/packages/43/ec/f9f61460adf4e16bfe64c59a8e708e2209521cd48d6ad6d8b1e14e7627f1/google_api_python_client-2.157.0.tar.gz", hash = "sha256:2ee342d0967ad1cedec43ccd7699671d94bff151e1f06833ea81303f9a6d86fd", size = 12275652 } wheels = [ - { url = "https://files.pythonhosted.org/packages/51/ab/d0671375afe79e6e8c51736e115a69bb6b4bcdc80cd5c01bf667486cd24c/google_api_python_client-2.159.0-py2.py3-none-any.whl", hash = "sha256:baef0bb631a60a0bd7c0bf12a5499e3a40cd4388484de7ee55c1950bf820a0cf", size = 12814228 }, + { url = "https://files.pythonhosted.org/packages/16/33/be58f58b63ffcc6b57e52428b388dbc94fb008baae60e81b205ea64e5baa/google_api_python_client-2.157.0-py2.py3-none-any.whl", hash = "sha256:0b0231db106324c659bf8b85f390391c00da57a60ebc4271e33def7aac198c75", size = 12787473 }, ] [[package]] @@ -1250,7 +1250,7 @@ wheels = [ [[package]] name = "google-cloud-aiplatform" -version = "1.77.0" +version = "1.75.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docstring-parser", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -1266,9 +1266,9 @@ dependencies = [ { name = "shapely", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4d/45/7ffd099ff7554d9f4f3665611afb44d3ea59f8a3dd071e4284381d0ac3c1/google_cloud_aiplatform-1.77.0.tar.gz", hash = "sha256:1e5b77fe6c7f276d7aae65bcf08a273122a71f6c4af1f43cf45821f603a74080", size = 8287282 } +sdist = { url = "https://files.pythonhosted.org/packages/de/76/7b3c013e92c70a558e71b0e83be13111ec797c4ded8ca98df20af15891c7/google_cloud_aiplatform-1.75.0.tar.gz", hash = "sha256:eb8404abf1134b3b368535fe429c4eec2fd12d444c2e9ffbc329ddcbc72b36c9", size = 8185280 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7b/b6/f7a3c8bdb08a3636d216c49768eff3369b5475edd71f6dbe590a942252b9/google_cloud_aiplatform-1.77.0-py2.py3-none-any.whl", hash = "sha256:e9dd1bcb1b9a85eddd452916cd6ad1d9ce2d487772a9e45b1814aa0ac5633689", size = 6939280 }, + { url = "https://files.pythonhosted.org/packages/06/d4/4b9df013c442e3b8db425924e896b5eaaeb23d1a036aa01002a3f83b936c/google_cloud_aiplatform-1.75.0-py2.py3-none-any.whl", hash = "sha256:eb5d79b5f7210d79a22b53c93a69b5bae5680dfc829387ea020765b97786b3d0", size = 6854342 }, ] [[package]] @@ -2812,15 +2812,15 @@ wheels = [ [[package]] name = "ollama" -version = "0.4.6" +version = "0.4.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "httpx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/75/d6/2bd7cffbabc81282576051ebf66ebfaa97e6b541975cd4e886bfd6c0f83d/ollama-0.4.6.tar.gz", hash = "sha256:b00717651c829f96094ed4231b9f0d87e33cc92dc235aca50aeb5a2a4e6e95b7", size = 12710 } +sdist = { url = "https://files.pythonhosted.org/packages/16/fd/a130173a62fd6dc7f7875919593b1e7a47bf3870a913c35d51ea013cfe41/ollama-0.4.5.tar.gz", hash = "sha256:e7fb71a99147046d028ab8b75e51e09437099aea6f8f9a0d91a71f787e97439e", size = 13104 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4a/60/ac0e47c4c400fbd1a72a3c6e4a76cf5ef859d60677e7c4b9f0203c5657d3/ollama-0.4.6-py3-none-any.whl", hash = "sha256:cbb4ebe009e10dd12bdd82508ab415fd131945e185753d728a7747c9ebe762e9", size = 13086 }, + { url = "https://files.pythonhosted.org/packages/93/71/44e508b6be7cc12efc498217bf74f443dbc1a31b145c87421d20fe61b70b/ollama-0.4.5-py3-none-any.whl", hash = "sha256:74936de89a41c87c9745f09f2e1db964b4783002188ac21241bfab747f46d925", size = 13205 }, ] [[package]] @@ -2886,7 +2886,7 @@ wheels = [ [[package]] name = "openai" -version = "1.59.7" +version = "1.59.3" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -2898,9 +2898,14 @@ dependencies = [ { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f9/d5/25cf04789c7929b476c4d9ef711f8979091db63d30bfc093828fe4bf5c72/openai-1.59.7.tar.gz", hash = "sha256:043603def78c00befb857df9f0a16ee76a3af5984ba40cb7ee5e2f40db4646bf", size = 345007 } +sdist = { url = "https://files.pythonhosted.org/packages/73/d0/def3c7620e1cb446947f098aeac9d88fc826b1760d66da279e4712d37666/openai-1.59.3.tar.gz", hash = "sha256:7f7fff9d8729968588edf1524e73266e8593bb6cab09298340efb755755bb66f", size = 344192 } wheels = [ - { url = "https://files.pythonhosted.org/packages/6d/47/7b92f1731c227f4139ef0025b5996062e44f9a749c54315c8bdb34bad5ec/openai-1.59.7-py3-none-any.whl", hash = "sha256:cfa806556226fa96df7380ab2e29814181d56fea44738c2b0e581b462c268692", size = 454844 }, + { url = "https://files.pythonhosted.org/packages/c7/26/0e0fb582bcb2a7cb6802447a749a2fc938fe4b82324097abccb86abfd5d1/openai-1.59.3-py3-none-any.whl", hash = "sha256:b041887a0d8f3e70d1fc6ffbb2bf7661c3b9a2f3e806c04bf42f572b9ac7bc37", size = 454793 }, +] + +[package.optional-dependencies] +realtime = [ + { name = "websockets", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] [[package]] @@ -3091,58 +3096,58 @@ wheels = [ [[package]] name = "orjson" -version = "3.10.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/f7/3219b56f47b4f5e864fb11cdf4ac0aaa3de608730ad2dc4c6e16382f35ec/orjson-3.10.14.tar.gz", hash = "sha256:cf31f6f071a6b8e7aa1ead1fa27b935b48d00fbfa6a28ce856cfff2d5dd68eed", size = 5282116 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b3/62/64348b8b29a14c7342f6aa45c8be0a87fdda2ce7716bc123717376537077/orjson-3.10.14-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:849ea7845a55f09965826e816cdc7689d6cf74fe9223d79d758c714af955bcb6", size = 249439 }, - { url = "https://files.pythonhosted.org/packages/9f/51/48f4dfbca7b4db630316b170db4a150a33cd405650258bd62a2d619b43b4/orjson-3.10.14-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5947b139dfa33f72eecc63f17e45230a97e741942955a6c9e650069305eb73d", size = 135811 }, - { url = "https://files.pythonhosted.org/packages/a1/1c/e18770843e6d045605c8e00a1be801da5668fa934b323b0492a49c9dee4f/orjson-3.10.14-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cde6d76910d3179dae70f164466692f4ea36da124d6fb1a61399ca589e81d69a", size = 150154 }, - { url = "https://files.pythonhosted.org/packages/51/1e/3817dc79164f1fc17fc53102f74f62d31f5f4ec042abdd24d94c5e06e51c/orjson-3.10.14-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6dfbaeb7afa77ca608a50e2770a0461177b63a99520d4928e27591b142c74b1", size = 139740 }, - { url = "https://files.pythonhosted.org/packages/ff/fc/fbf9e25448f7a2d67c1a2b6dad78a9340666bf9fda3339ff59b1e93f0b6f/orjson-3.10.14-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa45e489ef80f28ff0e5ba0a72812b8cfc7c1ef8b46a694723807d1b07c89ebb", size = 154479 }, - { url = "https://files.pythonhosted.org/packages/d4/df/c8b7ea21ff658f6a9a26d562055631c01d445bda5eb613c02c7d0934607d/orjson-3.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5007abfdbb1d866e2aa8990bd1c465f0f6da71d19e695fc278282be12cffa5", size = 130414 }, - { url = "https://files.pythonhosted.org/packages/df/f7/e29c2d42bef8fbf696a5e54e6339b0b9ea5179326950fee6ae80acf59d09/orjson-3.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1b49e2af011c84c3f2d541bb5cd1e3c7c2df672223e7e3ea608f09cf295e5f8a", size = 138545 }, - { url = "https://files.pythonhosted.org/packages/8e/97/afdf2908fe8eaeecb29e97fa82dc934f275acf330e5271def0b8fbac5478/orjson-3.10.14-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:164ac155109226b3a2606ee6dda899ccfbe6e7e18b5bdc3fbc00f79cc074157d", size = 130952 }, - { url = "https://files.pythonhosted.org/packages/4a/dd/04e01c1305694f47e9794c60ec7cece02e55fa9d57c5d72081eaaa62ad1d/orjson-3.10.14-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:6b1225024cf0ef5d15934b5ffe9baf860fe8bc68a796513f5ea4f5056de30bca", size = 414673 }, - { url = "https://files.pythonhosted.org/packages/fa/12/28c4d5f6a395ac9693b250f0662366968c47fc99c8f3cd803a65b1f5ba46/orjson-3.10.14-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d6546e8073dc382e60fcae4a001a5a1bc46da5eab4a4878acc2d12072d6166d5", size = 141002 }, - { url = "https://files.pythonhosted.org/packages/21/f6/357cb167c2d2fd9542251cfd9f68681b67ed4dcdac82aa6ee2f4f3ab952e/orjson-3.10.14-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f1d2942605c894162252d6259b0121bf1cb493071a1ea8cb35d79cb3e6ac5bc", size = 129626 }, - { url = "https://files.pythonhosted.org/packages/df/07/d9062353500df9db8bfa7c6a5982687c97d0b69a5b158c4166d407ac94e2/orjson-3.10.14-cp310-cp310-win32.whl", hash = "sha256:397083806abd51cf2b3bbbf6c347575374d160331a2d33c5823e22249ad3118b", size = 142429 }, - { url = "https://files.pythonhosted.org/packages/50/ba/6ba2bf69ac0526d143aebe78bc39e6e5fbb51d5336fbc5efb9aab6687cd9/orjson-3.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:fa18f949d3183a8d468367056be989666ac2bef3a72eece0bade9cdb733b3c28", size = 133512 }, - { url = "https://files.pythonhosted.org/packages/bf/18/26721760368e12b691fb6811692ed21ae5275ea918db409ba26866cacbe8/orjson-3.10.14-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f506fd666dd1ecd15a832bebc66c4df45c1902fd47526292836c339f7ba665a9", size = 249437 }, - { url = "https://files.pythonhosted.org/packages/d5/5b/2adfe7cc301edeb3bffc1942956659c19ec00d51a21c53c17c0767bebf47/orjson-3.10.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efe5fd254cfb0eeee13b8ef7ecb20f5d5a56ddda8a587f3852ab2cedfefdb5f6", size = 135812 }, - { url = "https://files.pythonhosted.org/packages/8a/68/07df7787fd9ff6dba815b2d793eec5e039d288fdf150431ed48a660bfcbb/orjson-3.10.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ddc8c866d7467f5ee2991397d2ea94bcf60d0048bdd8ca555740b56f9042725", size = 150153 }, - { url = "https://files.pythonhosted.org/packages/02/71/f68562734461b801b53bacd5365e079dcb3c78656a662f0639494880e522/orjson-3.10.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af8e42ae4363773658b8d578d56dedffb4f05ceeb4d1d4dd3fb504950b45526", size = 139742 }, - { url = "https://files.pythonhosted.org/packages/04/03/1355fb27652582f00d3c62e93a32b982fa42bc31d2e07f0a317867069096/orjson-3.10.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84dd83110503bc10e94322bf3ffab8bc49150176b49b4984dc1cce4c0a993bf9", size = 154479 }, - { url = "https://files.pythonhosted.org/packages/7c/47/1c2a840f27715e8bc2bbafffc851512ede6e53483593eded190919bdcaf4/orjson-3.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36f5bfc0399cd4811bf10ec7a759c7ab0cd18080956af8ee138097d5b5296a95", size = 130413 }, - { url = "https://files.pythonhosted.org/packages/dd/b2/5bb51006cbae85b052d1bbee7ff43ae26fa155bb3d31a71b0c07d384d5e3/orjson-3.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:868943660fb2a1e6b6b965b74430c16a79320b665b28dd4511d15ad5038d37d5", size = 138545 }, - { url = "https://files.pythonhosted.org/packages/79/30/7841a5dd46bb46b8e868791d5469c9d4788d3e26b7e69d40256647997baf/orjson-3.10.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:33449c67195969b1a677533dee9d76e006001213a24501333624623e13c7cc8e", size = 130953 }, - { url = "https://files.pythonhosted.org/packages/08/49/720e7c2040c0f1df630a36d83d449bd7e4d4471071d5ece47a4f7211d570/orjson-3.10.14-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e4c9f60f9fb0b5be66e416dcd8c9d94c3eabff3801d875bdb1f8ffc12cf86905", size = 414675 }, - { url = "https://files.pythonhosted.org/packages/50/b0/ca7619f34280e7dcbd50dbc9c5fe5200c12cd7269b8858652beb3887483f/orjson-3.10.14-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0de4d6315cfdbd9ec803b945c23b3a68207fd47cbe43626036d97e8e9561a436", size = 141004 }, - { url = "https://files.pythonhosted.org/packages/75/1b/7548e3a711543f438e87a4349e00439ab7f37807942e5659f29363f35765/orjson-3.10.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83adda3db595cb1a7e2237029b3249c85afbe5c747d26b41b802e7482cb3933e", size = 129629 }, - { url = "https://files.pythonhosted.org/packages/b0/1e/4930a6ff46debd6be1ff18e869b7bc43a7ad762c865610b7e745038d6f68/orjson-3.10.14-cp311-cp311-win32.whl", hash = "sha256:998019ef74a4997a9d741b1473533cdb8faa31373afc9849b35129b4b8ec048d", size = 142430 }, - { url = "https://files.pythonhosted.org/packages/28/e0/6cc1cd1dfde36555e81ac869f7847e86bb11c27f97b72fde2f1509b12163/orjson-3.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:9d034abdd36f0f0f2240f91492684e5043d46f290525d1117712d5b8137784eb", size = 133516 }, - { url = "https://files.pythonhosted.org/packages/8c/dc/dc5a882be016ee8688bd867ad3b4e3b2ab039d91383099702301a1adb6ac/orjson-3.10.14-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:2ad4b7e367efba6dc3f119c9a0fcd41908b7ec0399a696f3cdea7ec477441b09", size = 249396 }, - { url = "https://files.pythonhosted.org/packages/f0/95/4c23ff5c0505cd687928608e0b7910ccb44ce59490079e1c17b7610aa0d0/orjson-3.10.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f496286fc85e93ce0f71cc84fc1c42de2decf1bf494094e188e27a53694777a7", size = 135689 }, - { url = "https://files.pythonhosted.org/packages/ad/39/b4bdd19604dce9d6509c4d86e8e251a1373a24204b4c4169866dcecbe5f5/orjson-3.10.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c7f189bbfcded40e41a6969c1068ba305850ba016665be71a217918931416fbf", size = 150136 }, - { url = "https://files.pythonhosted.org/packages/1d/92/7b9bad96353abd3e89947960252dcf1022ce2df7f29056e434de05e18b6d/orjson-3.10.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8cc8204f0b75606869c707da331058ddf085de29558b516fc43c73ee5ee2aadb", size = 139766 }, - { url = "https://files.pythonhosted.org/packages/a6/bd/abb13c86540b7a91b40d7d9f8549d03a026bc22d78fa93f71d68b8f4c36e/orjson-3.10.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:deaa2899dff7f03ab667e2ec25842d233e2a6a9e333efa484dfe666403f3501c", size = 154533 }, - { url = "https://files.pythonhosted.org/packages/c0/02/0bcb91ec9c7143012359983aca44f567f87df379957cd4af11336217b12f/orjson-3.10.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1c3ea52642c9714dc6e56de8a451a066f6d2707d273e07fe8a9cc1ba073813d", size = 130658 }, - { url = "https://files.pythonhosted.org/packages/b4/1e/b304596bb1f800d47d6e92305bd09f0eef693ed4f7b2095db63f9808b229/orjson-3.10.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9d3f9ed72e7458ded9a1fb1b4d4ed4c4fdbaf82030ce3f9274b4dc1bff7ace2b", size = 138546 }, - { url = "https://files.pythonhosted.org/packages/56/c7/65d72b22080186ef618a46afeb9386e20056f3237664090f3a2f8da1cd6d/orjson-3.10.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:07520685d408a2aba514c17ccc16199ff2934f9f9e28501e676c557f454a37fe", size = 130774 }, - { url = "https://files.pythonhosted.org/packages/4d/85/1ab35a832f32b37ccd673721e845cf302f23453603112255af611c91d1d1/orjson-3.10.14-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:76344269b550ea01488d19a2a369ab572c1ac4449a72e9f6ac0d70eb1cbfb953", size = 414649 }, - { url = "https://files.pythonhosted.org/packages/d1/7d/1d6575f779bab8fe698fa6d52e8aa3aa0a9fca4885d0bf6197700455713a/orjson-3.10.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e2979d0f2959990620f7e62da6cd954e4620ee815539bc57a8ae46e2dacf90e3", size = 141060 }, - { url = "https://files.pythonhosted.org/packages/f8/26/68513e28b3bd1d7633318ed2818e86d1bfc8b782c87c520c7b363092837f/orjson-3.10.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03f61ca3674555adcb1aa717b9fc87ae936aa7a63f6aba90a474a88701278780", size = 129798 }, - { url = "https://files.pythonhosted.org/packages/44/ca/020fb99c98ff7267ba18ce798ff0c8c3aa97cd949b611fc76cad3c87e534/orjson-3.10.14-cp312-cp312-win32.whl", hash = "sha256:d5075c54edf1d6ad81d4c6523ce54a748ba1208b542e54b97d8a882ecd810fd1", size = 142524 }, - { url = "https://files.pythonhosted.org/packages/70/7f/f2d346819a273653825e7c92dc26418c8da506003c9fc1dfe8157e733b2e/orjson-3.10.14-cp312-cp312-win_amd64.whl", hash = "sha256:175cafd322e458603e8ce73510a068d16b6e6f389c13f69bf16de0e843d7d406", size = 133663 }, - { url = "https://files.pythonhosted.org/packages/46/bb/f1b037d89f580c79eda0940772384cc226a697be1cb4eb94ae4e792aa34c/orjson-3.10.14-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:0905ca08a10f7e0e0c97d11359609300eb1437490a7f32bbaa349de757e2e0c7", size = 249333 }, - { url = "https://files.pythonhosted.org/packages/e4/72/12958a073cace3f8acef0f9a30739d95f46bbb1544126fecad11527d4508/orjson-3.10.14-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92d13292249f9f2a3e418cbc307a9fbbef043c65f4bd8ba1eb620bc2aaba3d15", size = 125038 }, - { url = "https://files.pythonhosted.org/packages/c0/ae/461f78b1c98de1bc034af88bc21c6a792cc63373261fbc10a6ee560814fa/orjson-3.10.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90937664e776ad316d64251e2fa2ad69265e4443067668e4727074fe39676414", size = 130604 }, - { url = "https://files.pythonhosted.org/packages/ae/d2/17f50513f56bff7898840fddf7fb88f501305b9b2605d2793ff224789665/orjson-3.10.14-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9ed3d26c4cb4f6babaf791aa46a029265850e80ec2a566581f5c2ee1a14df4f1", size = 130756 }, - { url = "https://files.pythonhosted.org/packages/fa/bc/673856e4af94c9890dfd8e2054c05dc2ddc16d1728c2aa0c5bd198943105/orjson-3.10.14-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:56ee546c2bbe9599aba78169f99d1dc33301853e897dbaf642d654248280dc6e", size = 414613 }, - { url = "https://files.pythonhosted.org/packages/09/01/08c5b69b0756dd1790fcffa569d6a28dedcd7b97f825e4b46537b788908c/orjson-3.10.14-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:901e826cb2f1bdc1fcef3ef59adf0c451e8f7c0b5deb26c1a933fb66fb505eae", size = 141010 }, - { url = "https://files.pythonhosted.org/packages/5b/98/72883bb6cf88fd364996e62d2026622ca79bfb8dbaf96ccdd2018ada25b1/orjson-3.10.14-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26336c0d4b2d44636e1e1e6ed1002f03c6aae4a8a9329561c8883f135e9ff010", size = 129732 }, - { url = "https://files.pythonhosted.org/packages/e4/99/347418f7ef56dcb478ba131a6112b8ddd5b747942652b6e77a53155a7e21/orjson-3.10.14-cp313-cp313-win32.whl", hash = "sha256:e2bc525e335a8545c4e48f84dd0328bc46158c9aaeb8a1c2276546e94540ea3d", size = 142504 }, - { url = "https://files.pythonhosted.org/packages/59/ac/5e96cad01083015f7bfdb02ccafa489da8e6caa7f4c519e215f04d2bd856/orjson-3.10.14-cp313-cp313-win_amd64.whl", hash = "sha256:eca04dfd792cedad53dc9a917da1a522486255360cb4e77619343a20d9f35364", size = 133388 }, +version = "3.10.13" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/45/0b/8c7eaf1e2152f1e0fb28ae7b22e2b35a6b1992953a1ebe0371ba4d41d3ad/orjson-3.10.13.tar.gz", hash = "sha256:eb9bfb14ab8f68d9d9492d4817ae497788a15fd7da72e14dfabc289c3bb088ec", size = 5438389 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/7c/c4/67206a3cd1b677e2dc8d0de102bebc993ce083548542461e9fa397ce3e7c/orjson-3.10.13-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1232c5e873a4d1638ef957c5564b4b0d6f2a6ab9e207a9b3de9de05a09d1d920", size = 248733 }, + { url = "https://files.pythonhosted.org/packages/9f/c7/49202bcefb75c614d8f221845dd185d4e4dab1aace9a09e99a840dd22abb/orjson-3.10.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d26a0eca3035619fa366cbaf49af704c7cb1d4a0e6c79eced9f6a3f2437964b6", size = 136954 }, + { url = "https://files.pythonhosted.org/packages/87/6c/21518e60589c27cc4bc76156d1a0980fe2be7f5419f5269e800e2e5902bb/orjson-3.10.13-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d4b6acd7c9c829895e50d385a357d4b8c3fafc19c5989da2bae11783b0fd4977", size = 149101 }, + { url = "https://files.pythonhosted.org/packages/e3/88/5eac5856b28df0273ac07187cd20a0e6108799d9f5f3382e2dd1398ec1b3/orjson-3.10.13-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1884e53c6818686891cc6fc5a3a2540f2f35e8c76eac8dc3b40480fb59660b00", size = 140445 }, + { url = "https://files.pythonhosted.org/packages/a9/66/a6455588709b6d0cb4ebc95bc775c19c548d1d1e354bd10ad018123698a2/orjson-3.10.13-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a428afb5720f12892f64920acd2eeb4d996595bf168a26dd9190115dbf1130d", size = 156532 }, + { url = "https://files.pythonhosted.org/packages/c2/41/58f73d6656f1c9d6e736549f36066ce16ba91e33a639c8cca278af09baf3/orjson-3.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba5b13b8739ce5b630c65cb1c85aedbd257bcc2b9c256b06ab2605209af75a2e", size = 131261 }, + { url = "https://files.pythonhosted.org/packages/c9/7e/81ca17c438733741265e8ebfa3e5436aa4e61332f91ebdc11eff27c7b152/orjson-3.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cab83e67f6aabda1b45882254b2598b48b80ecc112968fc6483fa6dae609e9f0", size = 139822 }, + { url = "https://files.pythonhosted.org/packages/be/fc/b1d72a5f431fc5ae9edfa5bb41fb3b5e9532a4181c5268e67bc2717217bf/orjson-3.10.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:62c3cc00c7e776c71c6b7b9c48c5d2701d4c04e7d1d7cdee3572998ee6dc57cc", size = 131901 }, + { url = "https://files.pythonhosted.org/packages/31/f6/8cdcd06e0d4ee37eba1c7a6cd2c5a8798a3a533f9b17b5e48a2a7dcdf6c9/orjson-3.10.13-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:dc03db4922e75bbc870b03fc49734cefbd50fe975e0878327d200022210b82d8", size = 415733 }, + { url = "https://files.pythonhosted.org/packages/f1/37/0aec8417b5a18136651d57af7955a5991a80abca6356cd4dd04a869ee8e6/orjson-3.10.13-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:22f1c9a30b43d14a041a6ea190d9eca8a6b80c4beb0e8b67602c82d30d6eec3e", size = 142454 }, + { url = "https://files.pythonhosted.org/packages/b7/06/679318d8da3ce897b1d0518073abe6b762e7994b4f765b959b39a7d909a4/orjson-3.10.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b42f56821c29e697c68d7d421410d7c1d8f064ae288b525af6a50cf99a4b1200", size = 130672 }, + { url = "https://files.pythonhosted.org/packages/90/e4/3d0018b3aee93385393b37af000214b18c6873bb0d0097ba1355b7cb23d2/orjson-3.10.13-cp310-cp310-win32.whl", hash = "sha256:0dbf3b97e52e093d7c3e93eb5eb5b31dc7535b33c2ad56872c83f0160f943487", size = 143675 }, + { url = "https://files.pythonhosted.org/packages/30/f1/3608a164a4fea07b795ace71862375e2c1686537d8f907d4c9f6f1d63008/orjson-3.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:46c249b4e934453be4ff2e518cd1adcd90467da7391c7a79eaf2fbb79c51e8c7", size = 135084 }, + { url = "https://files.pythonhosted.org/packages/01/44/7a047e47779953e3f657a612ad36f71a0bca02cf57ff490c427e22b01833/orjson-3.10.13-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a36c0d48d2f084c800763473020a12976996f1109e2fcb66cfea442fdf88047f", size = 248732 }, + { url = "https://files.pythonhosted.org/packages/d6/e9/54976977aaacc5030fdd8012479638bb8d4e2a16519b516ac2bd03a48eab/orjson-3.10.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0065896f85d9497990731dfd4a9991a45b0a524baec42ef0a63c34630ee26fd6", size = 136954 }, + { url = "https://files.pythonhosted.org/packages/7f/a7/663fb04e031d5c80a348aeb7271c6042d13f80393c4951b8801a703b89c0/orjson-3.10.13-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92b4ec30d6025a9dcdfe0df77063cbce238c08d0404471ed7a79f309364a3d19", size = 149101 }, + { url = "https://files.pythonhosted.org/packages/f9/f1/5f2a4bf7525ef4acf48902d2df2bcc1c5aa38f6cc17ee0729a1d3e110ddb/orjson-3.10.13-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a94542d12271c30044dadad1125ee060e7a2048b6c7034e432e116077e1d13d2", size = 140445 }, + { url = "https://files.pythonhosted.org/packages/12/d3/e68afa1db9860880e59260348b54c0518d8dfe2297e932f8e333ace878fa/orjson-3.10.13-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3723e137772639af8adb68230f2aa4bcb27c48b3335b1b1e2d49328fed5e244c", size = 156530 }, + { url = "https://files.pythonhosted.org/packages/77/ee/492b198c77b9985ae28e0c6b8092c2994cd18d6be40dc7cb7f9a385b7096/orjson-3.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f00c7fb18843bad2ac42dc1ce6dd214a083c53f1e324a0fd1c8137c6436269b", size = 131260 }, + { url = "https://files.pythonhosted.org/packages/57/d2/5167cc1ccbe56bacdd9fc79e6a3276cba6aa90057305e8485db58b8250c4/orjson-3.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0e2759d3172300b2f892dee85500b22fca5ac49e0c42cfff101aaf9c12ac9617", size = 139821 }, + { url = "https://files.pythonhosted.org/packages/74/f0/c1cf568e0f90d812e00c77da2db04a13e94afe639665b9a09c271456dc41/orjson-3.10.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee948c6c01f6b337589c88f8e0bb11e78d32a15848b8b53d3f3b6fea48842c12", size = 131904 }, + { url = "https://files.pythonhosted.org/packages/55/7d/a611542afbbacca4693a2319744944134df62957a1f206303d5b3160e349/orjson-3.10.13-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa6fe68f0981fba0d4bf9cdc666d297a7cdba0f1b380dcd075a9a3dd5649a69e", size = 415733 }, + { url = "https://files.pythonhosted.org/packages/64/3f/e8182716695cd8d5ebec49d283645b8c7b1de7ed1c27db2891b6957e71f6/orjson-3.10.13-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dbcd7aad6bcff258f6896abfbc177d54d9b18149c4c561114f47ebfe74ae6bfd", size = 142456 }, + { url = "https://files.pythonhosted.org/packages/dc/10/e4b40f15be7e4e991737d77062399c7f67da9b7e3bc28bbcb25de1717df3/orjson-3.10.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2149e2fcd084c3fd584881c7f9d7f9e5ad1e2e006609d8b80649655e0d52cd02", size = 130676 }, + { url = "https://files.pythonhosted.org/packages/ad/b1/8b9fb36d470fe8ff99727972c77846673ebc962cb09a5af578804f9f2408/orjson-3.10.13-cp311-cp311-win32.whl", hash = "sha256:89367767ed27b33c25c026696507c76e3d01958406f51d3a2239fe9e91959df2", size = 143672 }, + { url = "https://files.pythonhosted.org/packages/b5/15/90b3711f40d27aff80dd42c1eec2f0ed704a1fa47eef7120350e2797892d/orjson-3.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:dca1d20f1af0daff511f6e26a27354a424f0b5cf00e04280279316df0f604a6f", size = 135082 }, + { url = "https://files.pythonhosted.org/packages/35/84/adf8842cf36904e6200acff76156862d48d39705054c1e7c5fa98fe14417/orjson-3.10.13-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a3614b00621c77f3f6487792238f9ed1dd8a42f2ec0e6540ee34c2d4e6db813a", size = 248778 }, + { url = "https://files.pythonhosted.org/packages/69/2f/22ac0c5f46748e9810287a5abaeabdd67f1120a74140db7d529582c92342/orjson-3.10.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c976bad3996aa027cd3aef78aa57873f3c959b6c38719de9724b71bdc7bd14b", size = 136759 }, + { url = "https://files.pythonhosted.org/packages/39/67/6f05de77dd383cb623e2807bceae13f136e9f179cd32633b7a27454e953f/orjson-3.10.13-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f74d878d1efb97a930b8a9f9898890067707d683eb5c7e20730030ecb3fb930", size = 149123 }, + { url = "https://files.pythonhosted.org/packages/f8/5c/b5e144e9adbb1dc7d1fdf54af9510756d09b65081806f905d300a926a755/orjson-3.10.13-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33ef84f7e9513fb13b3999c2a64b9ca9c8143f3da9722fbf9c9ce51ce0d8076e", size = 140557 }, + { url = "https://files.pythonhosted.org/packages/91/fd/7bdbc0aa374d49cdb917ee51c80851c99889494be81d5e7ec9f5f9cbe149/orjson-3.10.13-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd2bcde107221bb9c2fa0c4aaba735a537225104173d7e19cf73f70b3126c993", size = 156626 }, + { url = "https://files.pythonhosted.org/packages/48/90/e583d6e29937ec30a164f1d86a0439c1a2477b5aae9f55d94b37a4f5b5f0/orjson-3.10.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:064b9dbb0217fd64a8d016a8929f2fae6f3312d55ab3036b00b1d17399ab2f3e", size = 131551 }, + { url = "https://files.pythonhosted.org/packages/47/0b/838c00ec7f048527aa0382299cd178bbe07c2cb1024b3111883e85d56d1f/orjson-3.10.13-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0044b0b8c85a565e7c3ce0a72acc5d35cda60793edf871ed94711e712cb637d", size = 139790 }, + { url = "https://files.pythonhosted.org/packages/ac/90/df06ac390f319a61d55a7a4efacb5d7082859f6ea33f0fdd5181ad0dde0c/orjson-3.10.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7184f608ad563032e398f311910bc536e62b9fbdca2041be889afcbc39500de8", size = 131717 }, + { url = "https://files.pythonhosted.org/packages/ea/68/eafb5e2fc84aafccfbd0e9e0552ff297ef5f9b23c7f2600cc374095a50de/orjson-3.10.13-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d36f689e7e1b9b6fb39dbdebc16a6f07cbe994d3644fb1c22953020fc575935f", size = 415690 }, + { url = "https://files.pythonhosted.org/packages/b8/cf/aa93b48801b2e42da223ef5a99b3e4970b02e7abea8509dd2a6a083e27fa/orjson-3.10.13-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:54433e421618cd5873e51c0e9d0b9fb35f7bf76eb31c8eab20b3595bb713cd3d", size = 142396 }, + { url = "https://files.pythonhosted.org/packages/8b/50/fb1a7060b79231c60a688037c2c8e9fe289b5a4378ec1f32cf8d33d9adf8/orjson-3.10.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e1ba0c5857dd743438acecc1cd0e1adf83f0a81fee558e32b2b36f89e40cee8b", size = 130842 }, + { url = "https://files.pythonhosted.org/packages/94/e6/44067052e28a13176da874ca53419b43cf0f6f01f4bf0539f2f70d8eacf6/orjson-3.10.13-cp312-cp312-win32.whl", hash = "sha256:a42b9fe4b0114b51eb5cdf9887d8c94447bc59df6dbb9c5884434eab947888d8", size = 143773 }, + { url = "https://files.pythonhosted.org/packages/f2/7d/510939d1b7f8ba387849e83666e898f214f38baa46c5efde94561453974d/orjson-3.10.13-cp312-cp312-win_amd64.whl", hash = "sha256:3a7df63076435f39ec024bdfeb4c9767ebe7b49abc4949068d61cf4857fa6d6c", size = 135234 }, + { url = "https://files.pythonhosted.org/packages/ef/42/482fced9a135c798f31e1088f608fa16735fdc484eb8ffdd29aa32d4e842/orjson-3.10.13-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:2cdaf8b028a976ebab837a2c27b82810f7fc76ed9fb243755ba650cc83d07730", size = 248726 }, + { url = "https://files.pythonhosted.org/packages/00/e7/6345653906ee6d2d6eabb767cdc4482c7809572dbda59224f40e48931efa/orjson-3.10.13-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48a946796e390cbb803e069472de37f192b7a80f4ac82e16d6eb9909d9e39d56", size = 126032 }, + { url = "https://files.pythonhosted.org/packages/ad/b8/0d2a2c739458ff7f9917a132225365d72d18f4b65c50cb8ebb5afb6fe184/orjson-3.10.13-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d64f1db5ecbc21eb83097e5236d6ab7e86092c1cd4c216c02533332951afc", size = 131547 }, + { url = "https://files.pythonhosted.org/packages/8d/ac/a1dc389cf364d576cf587a6f78dac6c905c5cac31b9dbd063bbb24335bf7/orjson-3.10.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:711878da48f89df194edd2ba603ad42e7afed74abcd2bac164685e7ec15f96de", size = 131682 }, + { url = "https://files.pythonhosted.org/packages/43/6c/debab76b830aba6449ec8a75ac77edebb0e7decff63eb3ecfb2cf6340a2e/orjson-3.10.13-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:cf16f06cb77ce8baf844bc222dbcb03838f61d0abda2c3341400c2b7604e436e", size = 415621 }, + { url = "https://files.pythonhosted.org/packages/c2/32/106e605db5369a6717036065e2b41ac52bd0d2712962edb3e026a452dc07/orjson-3.10.13-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8257c3fb8dd7b0b446b5e87bf85a28e4071ac50f8c04b6ce2d38cb4abd7dff57", size = 142388 }, + { url = "https://files.pythonhosted.org/packages/a3/02/6b2103898d60c2565bf97abffdf3a4cf338920b9feb55eec1fd791ab10ee/orjson-3.10.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9c3a87abe6f849a4a7ac8a8a1dede6320a4303d5304006b90da7a3cd2b70d2c", size = 130825 }, + { url = "https://files.pythonhosted.org/packages/87/7c/db115e2380435da569732999d5c4c9b9868efe72e063493cb73c36bb649a/orjson-3.10.13-cp313-cp313-win32.whl", hash = "sha256:527afb6ddb0fa3fe02f5d9fba4920d9d95da58917826a9be93e0242da8abe94a", size = 143723 }, + { url = "https://files.pythonhosted.org/packages/cc/5e/c2b74a0b38ec561a322d8946663924556c1f967df2eefe1b9e0b98a33950/orjson-3.10.13-cp313-cp313-win_amd64.whl", hash = "sha256:b5f7c298d4b935b222f52d6c7f2ba5eafb59d690d9a3840b7b5c5cda97f6ec5c", size = 134968 }, ] [[package]] @@ -3396,7 +3401,7 @@ wheels = [ [[package]] name = "posthog" -version = "3.8.3" +version = "3.7.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "backoff", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -3405,9 +3410,9 @@ dependencies = [ { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "six", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/51/5a/057ebd6b279940e2cf2cbe8b10a4b34bc832f6f82b10649dcd12210219e9/posthog-3.8.3.tar.gz", hash = "sha256:263df03ea312d4b47a3d5ea393fdb22ff2ed78140d5ce9af9dd0618ae245a44b", size = 56864 } +sdist = { url = "https://files.pythonhosted.org/packages/58/e9/1cd7492bb58dd255129467e1221e2d6f51aa0c6f3c781ac9ac29cc8a2859/posthog-3.7.5.tar.gz", hash = "sha256:8ba40ab623da35db72715fc87fe7dccb7fc272ced92581fe31db2d4dbe7ad761", size = 50269 } wheels = [ - { url = "https://files.pythonhosted.org/packages/83/3a/ff36f067367de4477d114ab04f42d5830849bad1b0949eb70c9858cdb7e2/posthog-3.8.3-py2.py3-none-any.whl", hash = "sha256:7215c4d7649b0c87905b42f460403311564996d776ab48d39852f46539a50f22", size = 64665 }, + { url = "https://files.pythonhosted.org/packages/76/bd/2d550ac79443cdbb6a5a4193c37820f04df0499e1ecbe8e41c5462cf0c2d/posthog-3.7.5-py2.py3-none-any.whl", hash = "sha256:022132c17069dde03c5c5904e2ae1b9bd68d5059cbc5a8dffc5c1537a1b71cb5", size = 54882 }, ] [[package]] @@ -3541,16 +3546,16 @@ wheels = [ [[package]] name = "protobuf" -version = "5.29.3" +version = "5.29.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/d1/e0a911544ca9993e0f17ce6d3cc0932752356c1b0a834397f28e63479344/protobuf-5.29.3.tar.gz", hash = "sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620", size = 424945 } +sdist = { url = "https://files.pythonhosted.org/packages/a5/73/4e6295c1420a9d20c9c351db3a36109b4c9aa601916cb7c6871e3196a1ca/protobuf-5.29.2.tar.gz", hash = "sha256:b2cc8e8bb7c9326996f0e160137b0861f1a82162502658df2951209d0cb0309e", size = 424901 } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/7a/1e38f3cafa022f477ca0f57a1f49962f21ad25850c3ca0acd3b9d0091518/protobuf-5.29.3-cp310-abi3-win32.whl", hash = "sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888", size = 422708 }, - { url = "https://files.pythonhosted.org/packages/61/fa/aae8e10512b83de633f2646506a6d835b151edf4b30d18d73afd01447253/protobuf-5.29.3-cp310-abi3-win_amd64.whl", hash = "sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a", size = 434508 }, - { url = "https://files.pythonhosted.org/packages/dd/04/3eaedc2ba17a088961d0e3bd396eac764450f431621b58a04ce898acd126/protobuf-5.29.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e", size = 417825 }, - { url = "https://files.pythonhosted.org/packages/4f/06/7c467744d23c3979ce250397e26d8ad8eeb2bea7b18ca12ad58313c1b8d5/protobuf-5.29.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84", size = 319573 }, - { url = "https://files.pythonhosted.org/packages/a8/45/2ebbde52ad2be18d3675b6bee50e68cd73c9e0654de77d595540b5129df8/protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f", size = 319672 }, - { url = "https://files.pythonhosted.org/packages/fd/b2/ab07b09e0f6d143dfb839693aa05765257bceaa13d03bf1a696b78323e7a/protobuf-5.29.3-py3-none-any.whl", hash = "sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f", size = 172550 }, + { url = "https://files.pythonhosted.org/packages/f3/42/6db5387124708d619ffb990a846fb123bee546f52868039f8fa964c5bc54/protobuf-5.29.2-cp310-abi3-win32.whl", hash = "sha256:c12ba8249f5624300cf51c3d0bfe5be71a60c63e4dcf51ffe9a68771d958c851", size = 422697 }, + { url = "https://files.pythonhosted.org/packages/6c/38/2fcc968b377b531882d6ab2ac99b10ca6d00108394f6ff57c2395fb7baff/protobuf-5.29.2-cp310-abi3-win_amd64.whl", hash = "sha256:842de6d9241134a973aab719ab42b008a18a90f9f07f06ba480df268f86432f9", size = 434495 }, + { url = "https://files.pythonhosted.org/packages/cb/26/41debe0f6615fcb7e97672057524687ed86fcd85e3da3f031c30af8f0c51/protobuf-5.29.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a0c53d78383c851bfa97eb42e3703aefdc96d2036a41482ffd55dc5f529466eb", size = 417812 }, + { url = "https://files.pythonhosted.org/packages/e4/20/38fc33b60dcfb380507b99494aebe8c34b68b8ac7d32808c4cebda3f6f6b/protobuf-5.29.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:494229ecd8c9009dd71eda5fd57528395d1eacdf307dbece6c12ad0dd09e912e", size = 319562 }, + { url = "https://files.pythonhosted.org/packages/90/4d/c3d61e698e0e41d926dbff6aa4e57428ab1a6fc3b5e1deaa6c9ec0fd45cf/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b6b0d416bbbb9d4fbf9d0561dbfc4e324fd522f61f7af0fe0f282ab67b22477e", size = 319662 }, + { url = "https://files.pythonhosted.org/packages/f3/fd/c7924b4c2a1c61b8f4b64edd7a31ffacf63432135a2606f03a2f0d75a750/protobuf-5.29.2-py3-none-any.whl", hash = "sha256:fde4554c0e578a5a0bcc9a276339594848d1e89f9ea47b4427c80e5d72f90181", size = 172539 }, ] [[package]] @@ -3733,6 +3738,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, ] +[[package]] +name = "pyaudio" +version = "0.2.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/26/1d/8878c7752febb0f6716a7e1a52cb92ac98871c5aa522cba181878091607c/PyAudio-0.2.14.tar.gz", hash = "sha256:78dfff3879b4994d1f4fc6485646a57755c6ee3c19647a491f790a0895bd2f87", size = 47066 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/90/90/1553487277e6aa25c0b7c2c38709cdd2b49e11c66c0b25c6e8b7b6638c72/PyAudio-0.2.14-cp310-cp310-win32.whl", hash = "sha256:126065b5e82a1c03ba16e7c0404d8f54e17368836e7d2d92427358ad44fefe61", size = 144624 }, + { url = "https://files.pythonhosted.org/packages/27/bc/719d140ee63cf4b0725016531d36743a797ffdbab85e8536922902c9349a/PyAudio-0.2.14-cp310-cp310-win_amd64.whl", hash = "sha256:2a166fc88d435a2779810dd2678354adc33499e9d4d7f937f28b20cc55893e83", size = 164069 }, + { url = "https://files.pythonhosted.org/packages/7b/f0/b0eab89eafa70a86b7b566a4df2f94c7880a2d483aa8de1c77d335335b5b/PyAudio-0.2.14-cp311-cp311-win32.whl", hash = "sha256:506b32a595f8693811682ab4b127602d404df7dfc453b499c91a80d0f7bad289", size = 144624 }, + { url = "https://files.pythonhosted.org/packages/82/d8/f043c854aad450a76e476b0cf9cda1956419e1dacf1062eb9df3c0055abe/PyAudio-0.2.14-cp311-cp311-win_amd64.whl", hash = "sha256:bbeb01d36a2f472ae5ee5e1451cacc42112986abe622f735bb870a5db77cf903", size = 164070 }, + { url = "https://files.pythonhosted.org/packages/8d/45/8d2b76e8f6db783f9326c1305f3f816d4a12c8eda5edc6a2e1d03c097c3b/PyAudio-0.2.14-cp312-cp312-win32.whl", hash = "sha256:5fce4bcdd2e0e8c063d835dbe2860dac46437506af509353c7f8114d4bacbd5b", size = 144750 }, + { url = "https://files.pythonhosted.org/packages/b0/6a/d25812e5f79f06285767ec607b39149d02aa3b31d50c2269768f48768930/PyAudio-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:12f2f1ba04e06ff95d80700a78967897a489c05e093e3bffa05a84ed9c0a7fa3", size = 164126 }, + { url = "https://files.pythonhosted.org/packages/3a/77/66cd37111a87c1589b63524f3d3c848011d21ca97828422c7fde7665ff0d/PyAudio-0.2.14-cp313-cp313-win32.whl", hash = "sha256:95328285b4dab57ea8c52a4a996cb52be6d629353315be5bfda403d15932a497", size = 150982 }, + { url = "https://files.pythonhosted.org/packages/a5/8b/7f9a061c1cc2b230f9ac02a6003fcd14c85ce1828013aecbaf45aa988d20/PyAudio-0.2.14-cp313-cp313-win_amd64.whl", hash = "sha256:692d8c1446f52ed2662120bcd9ddcb5aa2b71f38bda31e58b19fb4672fffba69", size = 173655 }, +] + [[package]] name = "pybars4" version = "0.9.13" @@ -3753,16 +3774,16 @@ wheels = [ [[package]] name = "pydantic" -version = "2.10.5" +version = "2.10.4" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pydantic-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/6a/c7/ca334c2ef6f2e046b1144fe4bb2a5da8a4c574e7f2ebf7e16b34a6a2fa92/pydantic-2.10.5.tar.gz", hash = "sha256:278b38dbbaec562011d659ee05f63346951b3a248a6f3642e1bc68894ea2b4ff", size = 761287 } +sdist = { url = "https://files.pythonhosted.org/packages/70/7e/fb60e6fee04d0ef8f15e4e01ff187a196fa976eb0f0ab524af4599e5754c/pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06", size = 762094 } wheels = [ - { url = "https://files.pythonhosted.org/packages/58/26/82663c79010b28eddf29dcdd0ea723439535fa917fce5905885c0e9ba562/pydantic-2.10.5-py3-none-any.whl", hash = "sha256:4dd4e322dbe55472cb7ca7e73f4b63574eecccf2835ffa2af9021ce113c83c53", size = 431426 }, + { url = "https://files.pythonhosted.org/packages/f3/26/3e1bbe954fde7ee22a6e7d31582c642aad9e84ffe4b5fb61e63b87cd326f/pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d", size = 431765 }, ] [[package]] @@ -3853,13 +3874,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/46/93416fdae86d40879714f72956ac14df9c7b76f7d41a4d68aa9f71a0028b/pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd", size = 29718 }, ] +[[package]] +name = "pydub" +version = "0.25.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/fe/9a/e6bca0eed82db26562c73b5076539a4a08d3cffd19c3cc5913a3e61145fd/pydub-0.25.1.tar.gz", hash = "sha256:980a33ce9949cab2a569606b65674d748ecbca4f0796887fd6f46173a7b0d30f", size = 38326 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/a6/53/d78dc063216e62fc55f6b2eebb447f6a4b0a59f55c8406376f76bf959b08/pydub-0.25.1-py2.py3-none-any.whl", hash = "sha256:65617e33033874b59d87db603aa1ed450633288aefead953b30bded59cb599a6", size = 32327 }, +] + [[package]] name = "pygments" -version = "2.19.1" +version = "2.19.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } +sdist = { url = "https://files.pythonhosted.org/packages/d3/c0/9c9832e5be227c40e1ce774d493065f83a91d6430baa7e372094e9683a45/pygments-2.19.0.tar.gz", hash = "sha256:afc4146269910d4bdfabcd27c24923137a74d562a23a320a41a55ad303e19783", size = 4967733 } wheels = [ - { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, + { url = "https://files.pythonhosted.org/packages/20/dc/fde3e7ac4d279a331676829af4afafd113b34272393d73f610e8f0329221/pygments-2.19.0-py3-none-any.whl", hash = "sha256:4755e6e64d22161d5b61432c0600c923c5927214e7c956e31c23923c89251a9b", size = 1225305 }, ] [[package]] @@ -3999,14 +4029,14 @@ wheels = [ [[package]] name = "pytest-asyncio" -version = "0.25.2" +version = "0.25.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/72/df/adcc0d60f1053d74717d21d58c0048479e9cab51464ce0d2965b086bd0e2/pytest_asyncio-0.25.2.tar.gz", hash = "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f", size = 53950 } +sdist = { url = "https://files.pythonhosted.org/packages/4b/04/0477a4bdd176ad678d148c075f43620b3f7a060ff61c7da48500b1fa8a75/pytest_asyncio-0.25.1.tar.gz", hash = "sha256:79be8a72384b0c917677e00daa711e07db15259f4d23203c59012bcd989d4aee", size = 53760 } wheels = [ - { url = "https://files.pythonhosted.org/packages/61/d8/defa05ae50dcd6019a95527200d3b3980043df5aa445d40cb0ef9f7f98ab/pytest_asyncio-0.25.2-py3-none-any.whl", hash = "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075", size = 19400 }, + { url = "https://files.pythonhosted.org/packages/81/fb/efc7226b384befd98d0e00d8c4390ad57f33c8fde00094b85c5e07897def/pytest_asyncio-0.25.1-py3-none-any.whl", hash = "sha256:c84878849ec63ff2ca509423616e071ef9cd8cc93c053aa33b5b8fb70a990671", size = 19357 }, ] [[package]] @@ -4563,27 +4593,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.9.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/80/63/77ecca9d21177600f551d1c58ab0e5a0b260940ea7312195bd2a4798f8a8/ruff-0.9.2.tar.gz", hash = "sha256:b5eceb334d55fae5f316f783437392642ae18e16dcf4f1858d55d3c2a0f8f5d0", size = 3553799 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/af/b9/0e168e4e7fb3af851f739e8f07889b91d1a33a30fca8c29fa3149d6b03ec/ruff-0.9.2-py3-none-linux_armv6l.whl", hash = "sha256:80605a039ba1454d002b32139e4970becf84b5fee3a3c3bf1c2af6f61a784347", size = 11652408 }, - { url = "https://files.pythonhosted.org/packages/2c/22/08ede5db17cf701372a461d1cb8fdde037da1d4fa622b69ac21960e6237e/ruff-0.9.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b9aab82bb20afd5f596527045c01e6ae25a718ff1784cb92947bff1f83068b00", size = 11587553 }, - { url = "https://files.pythonhosted.org/packages/42/05/dedfc70f0bf010230229e33dec6e7b2235b2a1b8cbb2a991c710743e343f/ruff-0.9.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:fbd337bac1cfa96be615f6efcd4bc4d077edbc127ef30e2b8ba2a27e18c054d4", size = 11020755 }, - { url = "https://files.pythonhosted.org/packages/df/9b/65d87ad9b2e3def67342830bd1af98803af731243da1255537ddb8f22209/ruff-0.9.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82b35259b0cbf8daa22a498018e300b9bb0174c2bbb7bcba593935158a78054d", size = 11826502 }, - { url = "https://files.pythonhosted.org/packages/93/02/f2239f56786479e1a89c3da9bc9391120057fc6f4a8266a5b091314e72ce/ruff-0.9.2-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:8b6a9701d1e371bf41dca22015c3f89769da7576884d2add7317ec1ec8cb9c3c", size = 11390562 }, - { url = "https://files.pythonhosted.org/packages/c9/37/d3a854dba9931f8cb1b2a19509bfe59e00875f48ade632e95aefcb7a0aee/ruff-0.9.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9cc53e68b3c5ae41e8faf83a3b89f4a5d7b2cb666dff4b366bb86ed2a85b481f", size = 12548968 }, - { url = "https://files.pythonhosted.org/packages/fa/c3/c7b812bb256c7a1d5553433e95980934ffa85396d332401f6b391d3c4569/ruff-0.9.2-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:8efd9da7a1ee314b910da155ca7e8953094a7c10d0c0a39bfde3fcfd2a015684", size = 13187155 }, - { url = "https://files.pythonhosted.org/packages/bd/5a/3c7f9696a7875522b66aa9bba9e326e4e5894b4366bd1dc32aa6791cb1ff/ruff-0.9.2-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3292c5a22ea9a5f9a185e2d131dc7f98f8534a32fb6d2ee7b9944569239c648d", size = 12704674 }, - { url = "https://files.pythonhosted.org/packages/be/d6/d908762257a96ce5912187ae9ae86792e677ca4f3dc973b71e7508ff6282/ruff-0.9.2-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1a605fdcf6e8b2d39f9436d343d1f0ff70c365a1e681546de0104bef81ce88df", size = 14529328 }, - { url = "https://files.pythonhosted.org/packages/2d/c2/049f1e6755d12d9cd8823242fa105968f34ee4c669d04cac8cea51a50407/ruff-0.9.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c547f7f256aa366834829a08375c297fa63386cbe5f1459efaf174086b564247", size = 12385955 }, - { url = "https://files.pythonhosted.org/packages/91/5a/a9bdb50e39810bd9627074e42743b00e6dc4009d42ae9f9351bc3dbc28e7/ruff-0.9.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:d18bba3d3353ed916e882521bc3e0af403949dbada344c20c16ea78f47af965e", size = 11810149 }, - { url = "https://files.pythonhosted.org/packages/e5/fd/57df1a0543182f79a1236e82a79c68ce210efb00e97c30657d5bdb12b478/ruff-0.9.2-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:b338edc4610142355ccf6b87bd356729b62bf1bc152a2fad5b0c7dc04af77bfe", size = 11479141 }, - { url = "https://files.pythonhosted.org/packages/dc/16/bc3fd1d38974f6775fc152a0554f8c210ff80f2764b43777163c3c45d61b/ruff-0.9.2-py3-none-musllinux_1_2_i686.whl", hash = "sha256:492a5e44ad9b22a0ea98cf72e40305cbdaf27fac0d927f8bc9e1df316dcc96eb", size = 12014073 }, - { url = "https://files.pythonhosted.org/packages/47/6b/e4ca048a8f2047eb652e1e8c755f384d1b7944f69ed69066a37acd4118b0/ruff-0.9.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:af1e9e9fe7b1f767264d26b1075ac4ad831c7db976911fa362d09b2d0356426a", size = 12435758 }, - { url = "https://files.pythonhosted.org/packages/c2/40/4d3d6c979c67ba24cf183d29f706051a53c36d78358036a9cd21421582ab/ruff-0.9.2-py3-none-win32.whl", hash = "sha256:71cbe22e178c5da20e1514e1e01029c73dc09288a8028a5d3446e6bba87a5145", size = 9796916 }, - { url = "https://files.pythonhosted.org/packages/c3/ef/7f548752bdb6867e6939489c87fe4da489ab36191525fadc5cede2a6e8e2/ruff-0.9.2-py3-none-win_amd64.whl", hash = "sha256:c5e1d6abc798419cf46eed03f54f2e0c3adb1ad4b801119dedf23fcaf69b55b5", size = 10773080 }, - { url = "https://files.pythonhosted.org/packages/0e/4e/33df635528292bd2d18404e4daabcd74ca8a9853b2e1df85ed3d32d24362/ruff-0.9.2-py3-none-win_arm64.whl", hash = "sha256:a1b63fa24149918f8b37cef2ee6fff81f24f0d74b6f0bdc37bc3e1f2143e41c6", size = 10001738 }, +version = "0.8.6" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/da/00/089db7890ea3be5709e3ece6e46408d6f1e876026ec3fd081ee585fef209/ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5", size = 3473116 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/28/aa07903694637c2fa394a9f4fe93cf861ad8b09f1282fa650ef07ff9fe97/ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3", size = 10628735 }, + { url = "https://files.pythonhosted.org/packages/2b/43/827bb1448f1fcb0fb42e9c6edf8fb067ca8244923bf0ddf12b7bf949065c/ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1", size = 10386758 }, + { url = "https://files.pythonhosted.org/packages/df/93/fc852a81c3cd315b14676db3b8327d2bb2d7508649ad60bfdb966d60738d/ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807", size = 10007808 }, + { url = "https://files.pythonhosted.org/packages/94/e9/e0ed4af1794335fb280c4fac180f2bf40f6a3b859cae93a5a3ada27325ae/ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25", size = 10861031 }, + { url = "https://files.pythonhosted.org/packages/82/68/da0db02f5ecb2ce912c2bef2aa9fcb8915c31e9bc363969cfaaddbc4c1c2/ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d", size = 10388246 }, + { url = "https://files.pythonhosted.org/packages/ac/1d/b85383db181639019b50eb277c2ee48f9f5168f4f7c287376f2b6e2a6dc2/ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75", size = 11424693 }, + { url = "https://files.pythonhosted.org/packages/ac/b7/30bc78a37648d31bfc7ba7105b108cb9091cd925f249aa533038ebc5a96f/ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315", size = 12141921 }, + { url = "https://files.pythonhosted.org/packages/60/b3/ee0a14cf6a1fbd6965b601c88d5625d250b97caf0534181e151504498f86/ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188", size = 11692419 }, + { url = "https://files.pythonhosted.org/packages/ef/d6/c597062b2931ba3e3861e80bd2b147ca12b3370afc3889af46f29209037f/ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf", size = 12981648 }, + { url = "https://files.pythonhosted.org/packages/68/84/21f578c2a4144917985f1f4011171aeff94ab18dfa5303ac632da2f9af36/ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117", size = 11251801 }, + { url = "https://files.pythonhosted.org/packages/6c/aa/1ac02537c8edeb13e0955b5db86b5c050a1dcba54f6d49ab567decaa59c1/ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe", size = 10849857 }, + { url = "https://files.pythonhosted.org/packages/eb/00/020cb222252d833956cb3b07e0e40c9d4b984fbb2dc3923075c8f944497d/ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d", size = 10470852 }, + { url = "https://files.pythonhosted.org/packages/00/56/e6d6578202a0141cd52299fe5acb38b2d873565f4670c7a5373b637cf58d/ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a", size = 10972997 }, + { url = "https://files.pythonhosted.org/packages/be/31/dd0db1f4796bda30dea7592f106f3a67a8f00bcd3a50df889fbac58e2786/ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76", size = 11317760 }, + { url = "https://files.pythonhosted.org/packages/d4/70/cfcb693dc294e034c6fed837fa2ec98b27cc97a26db5d049345364f504bf/ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764", size = 8799729 }, + { url = "https://files.pythonhosted.org/packages/60/22/ae6bcaa0edc83af42751bd193138bfb7598b2990939d3e40494d6c00698c/ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905", size = 9673857 }, + { url = "https://files.pythonhosted.org/packages/91/f8/3765e053acd07baa055c96b2065c7fab91f911b3c076dfea71006666f5b0/ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162", size = 9149556 }, ] [[package]] @@ -4600,24 +4630,24 @@ wheels = [ [[package]] name = "safetensors" -version = "0.5.2" +version = "0.5.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f4/4f/2ef9ef1766f8c194b01b67a63a444d2e557c8fe1d82faf3ebd85f370a917/safetensors-0.5.2.tar.gz", hash = "sha256:cb4a8d98ba12fa016f4241932b1fc5e702e5143f5374bba0bbcf7ddc1c4cf2b8", size = 66957 } +sdist = { url = "https://files.pythonhosted.org/packages/5d/b3/1d9000e9d0470499d124ca63c6908f8092b528b48bd95ba11507e14d9dba/safetensors-0.5.0.tar.gz", hash = "sha256:c47b34c549fa1e0c655c4644da31332c61332c732c47c8dd9399347e9aac69d1", size = 65660 } wheels = [ - { url = "https://files.pythonhosted.org/packages/96/d1/017e31e75e274492a11a456a9e7c171f8f7911fe50735b4ec6ff37221220/safetensors-0.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:45b6092997ceb8aa3801693781a71a99909ab9cc776fbc3fa9322d29b1d3bef2", size = 427067 }, - { url = "https://files.pythonhosted.org/packages/24/84/e9d3ff57ae50dd0028f301c9ee064e5087fe8b00e55696677a0413c377a7/safetensors-0.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6d0d6a8ee2215a440e1296b843edf44fd377b055ba350eaba74655a2fe2c4bae", size = 408856 }, - { url = "https://files.pythonhosted.org/packages/f1/1d/fe95f5dd73db16757b11915e8a5106337663182d0381811c81993e0014a9/safetensors-0.5.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86016d40bcaa3bcc9a56cd74d97e654b5f4f4abe42b038c71e4f00a089c4526c", size = 450088 }, - { url = "https://files.pythonhosted.org/packages/cf/21/e527961b12d5ab528c6e47b92d5f57f33563c28a972750b238b871924e49/safetensors-0.5.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:990833f70a5f9c7d3fc82c94507f03179930ff7d00941c287f73b6fcbf67f19e", size = 458966 }, - { url = "https://files.pythonhosted.org/packages/a5/8b/1a037d7a57f86837c0b41905040369aea7d8ca1ec4b2a77592372b2ec380/safetensors-0.5.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dfa7c2f3fe55db34eba90c29df94bcdac4821043fc391cb5d082d9922013869", size = 509915 }, - { url = "https://files.pythonhosted.org/packages/61/3d/03dd5cfd33839df0ee3f4581a20bd09c40246d169c0e4518f20b21d5f077/safetensors-0.5.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46ff2116150ae70a4e9c490d2ab6b6e1b1b93f25e520e540abe1b81b48560c3a", size = 527664 }, - { url = "https://files.pythonhosted.org/packages/c5/dc/8952caafa9a10a3c0f40fa86bacf3190ae7f55fa5eef87415b97b29cb97f/safetensors-0.5.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab696dfdc060caffb61dbe4066b86419107a24c804a4e373ba59be699ebd8d5", size = 461978 }, - { url = "https://files.pythonhosted.org/packages/60/da/82de1fcf1194e3dbefd4faa92dc98b33c06bed5d67890e0962dd98e18287/safetensors-0.5.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975", size = 491253 }, - { url = "https://files.pythonhosted.org/packages/5a/9a/d90e273c25f90c3ba1b0196a972003786f04c39e302fbd6649325b1272bb/safetensors-0.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a00e737948791b94dad83cf0eafc09a02c4d8c2171a239e8c8572fe04e25960e", size = 628644 }, - { url = "https://files.pythonhosted.org/packages/70/3c/acb23e05aa34b4f5edd2e7f393f8e6480fbccd10601ab42cd03a57d4ab5f/safetensors-0.5.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d3a06fae62418ec8e5c635b61a8086032c9e281f16c63c3af46a6efbab33156f", size = 721648 }, - { url = "https://files.pythonhosted.org/packages/71/45/eaa3dba5253a7c6931230dc961641455710ab231f8a89cb3c4c2af70f8c8/safetensors-0.5.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:1506e4c2eda1431099cebe9abf6c76853e95d0b7a95addceaa74c6019c65d8cf", size = 659588 }, - { url = "https://files.pythonhosted.org/packages/b0/71/2f9851164f821064d43b481ddbea0149c2d676c4f4e077b178e7eeaa6660/safetensors-0.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5c5b5d9da594f638a259fca766046f44c97244cc7ab8bef161b3e80d04becc76", size = 632533 }, - { url = "https://files.pythonhosted.org/packages/00/f1/5680e2ef61d9c61454fad82c344f0e40b8741a9dbd1e31484f0d31a9b1c3/safetensors-0.5.2-cp38-abi3-win32.whl", hash = "sha256:fe55c039d97090d1f85277d402954dd6ad27f63034fa81985a9cc59655ac3ee2", size = 291167 }, - { url = "https://files.pythonhosted.org/packages/86/ca/aa489392ec6fb59223ffce825461e1f811a3affd417121a2088be7a5758b/safetensors-0.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:78abdddd03a406646107f973c7843276e7b64e5e32623529dc17f3d94a20f589", size = 303756 }, + { url = "https://files.pythonhosted.org/packages/0f/ee/0fd61b99bc58db736a3ab3d97d49d4a11afe71ee0aad85b25d6c4235b743/safetensors-0.5.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c683b9b485bee43422ba2855f72777c37647190281e03da4c8d2a69fa5336558", size = 426509 }, + { url = "https://files.pythonhosted.org/packages/51/aa/de1a11aa056d0241f95d5de9dbb1ac2dabaf3df5c568f9375451fd593c95/safetensors-0.5.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6106aa835deb7263f7014f74c05842ab828d6c11d789f2e7e98f26b1a305e72d", size = 408471 }, + { url = "https://files.pythonhosted.org/packages/a5/c7/84b821bd90547a909053a8526ff70446f062287cda20d0ec024c1a1f80f6/safetensors-0.5.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1349611f74f55c5ee1c1c144c536a2743c38f7d8bf60b9fc8267e0efc0591a2", size = 449638 }, + { url = "https://files.pythonhosted.org/packages/b5/25/3d20bb9f669fec704e01d70849e9c6c054601efe9b5e784ce9a865cf3c52/safetensors-0.5.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d936028ac799e18644b08a91fd98b4b62ae3dcd0440b1cfcb56535785589f1", size = 458246 }, + { url = "https://files.pythonhosted.org/packages/31/35/68e1c39c4ad6a2f9373fc89588c0fbd29b1899c57c3a6482fc8e42fa4c8f/safetensors-0.5.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f26afada2233576ffea6b80042c2c0a8105c164254af56168ec14299ad3122", size = 509573 }, + { url = "https://files.pythonhosted.org/packages/85/b0/79927c6d4f70232f04a46785ea8b0ed0f70f9be74d17e0a90e1890523553/safetensors-0.5.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20067e7a5e63f0cbc88457b2a1161e70ff73af4cc3a24bce90309430cd6f6e7e", size = 525555 }, + { url = "https://files.pythonhosted.org/packages/a6/83/ca8c1af662a20a545c174b8949e63865b747c180b607260eed83c1d38c72/safetensors-0.5.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649d6a4aa34d5174ae87289068ccc2fec2a1a998ecf83425aa5a42c3eff69bcf", size = 461294 }, + { url = "https://files.pythonhosted.org/packages/81/ef/1d11d08b14b36e3e3d701629c9685ad95c3afee7da2851658d6c65cad9be/safetensors-0.5.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:debff88f41d569a3e93a955469f83864e432af35bb34b16f65a9ddf378daa3ae", size = 490593 }, + { url = "https://files.pythonhosted.org/packages/f6/9a/50bf824a26d768d33485b7208ba5e6a173a80a2633be5e213a2494d1569b/safetensors-0.5.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bdf6a3e366ea8ba1a0538db6099229e95811194432c684ea28ea7ae28763b8dc", size = 628142 }, + { url = "https://files.pythonhosted.org/packages/28/22/dc5ae22523b8221017dbf6984fedfe2c6f35ff4cc76e80bbab2b9e14cc8a/safetensors-0.5.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0371afd84c200a80eb7103bf715108b0c3846132fb82453ae018609a15551580", size = 721377 }, + { url = "https://files.pythonhosted.org/packages/fe/87/36323e8058e7101ef0101fde6d71c375a9ab6059d3d9501fe8fb8d13a45a/safetensors-0.5.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5ec7fc8c3d2f32ebf1c7011bc886b362e53ee0a1ec6d828c39d531fed8b325d6", size = 659192 }, + { url = "https://files.pythonhosted.org/packages/dd/2f/8d526f06bb192b45b4e0fec94284d568497e6e19620c834373749a5f9787/safetensors-0.5.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:53715e4ea0ef23c08f004baae0f609a7773de7d4148727760417c6760cfd6b76", size = 632231 }, + { url = "https://files.pythonhosted.org/packages/d3/68/1166bba02f77c811d17766e54a54d7714c1276f54bfcf60d50bb9326a1b4/safetensors-0.5.0-cp38-abi3-win32.whl", hash = "sha256:b85565bc2f0456961a788d2f11d9d892eec46603db0e4923aa9512c2355aa727", size = 290608 }, + { url = "https://files.pythonhosted.org/packages/0c/ab/a428973e43a77791d2fd4b6425f4fd82e9f8559b32222c861acbbd7bc910/safetensors-0.5.0-cp38-abi3-win_amd64.whl", hash = "sha256:f451941f8aa11e7be5c3fa450e264609a2b1e65fa38ae590a74e55a94d646b76", size = 303322 }, ] [[package]] @@ -4660,52 +4690,52 @@ wheels = [ [[package]] name = "scipy" -version = "1.15.1" +version = "1.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "numpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/76/c6/8eb0654ba0c7d0bb1bf67bf8fbace101a8e4f250f7722371105e8b6f68fc/scipy-1.15.1.tar.gz", hash = "sha256:033a75ddad1463970c96a88063a1df87ccfddd526437136b6ee81ff0312ebdf6", size = 59407493 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/86/53/b204ce5a4433f1864001b9d16f103b9c25f5002a602ae83585d0ea5f9c4a/scipy-1.15.1-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:c64ded12dcab08afff9e805a67ff4480f5e69993310e093434b10e85dc9d43e1", size = 41414518 }, - { url = "https://files.pythonhosted.org/packages/c7/fc/54ffa7a8847f7f303197a6ba65a66104724beba2e38f328135a78f0dc480/scipy-1.15.1-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5b190b935e7db569960b48840e5bef71dc513314cc4e79a1b7d14664f57fd4ff", size = 32519265 }, - { url = "https://files.pythonhosted.org/packages/f1/77/a98b8ba03d6f371dc31a38719affd53426d4665729dcffbed4afe296784a/scipy-1.15.1-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:4b17d4220df99bacb63065c76b0d1126d82bbf00167d1730019d2a30d6ae01ea", size = 24792859 }, - { url = "https://files.pythonhosted.org/packages/a7/78/70bb9f0df7444b18b108580934bfef774822e28fd34a68e5c263c7d2828a/scipy-1.15.1-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:63b9b6cd0333d0eb1a49de6f834e8aeaefe438df8f6372352084535ad095219e", size = 27886506 }, - { url = "https://files.pythonhosted.org/packages/14/a7/f40f6033e06de4176ddd6cc8c3ae9f10a226c3bca5d6b4ab883bc9914a14/scipy-1.15.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9f151e9fb60fbf8e52426132f473221a49362091ce7a5e72f8aa41f8e0da4f25", size = 38375041 }, - { url = "https://files.pythonhosted.org/packages/17/03/390a1c5c61fd76b0fa4b3c5aa3bdd7e60f6c46f712924f1a9df5705ec046/scipy-1.15.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21e10b1dd56ce92fba3e786007322542361984f8463c6d37f6f25935a5a6ef52", size = 40597556 }, - { url = "https://files.pythonhosted.org/packages/4e/70/fa95b3ae026b97eeca58204a90868802e5155ac71b9d7bdee92b68115dd3/scipy-1.15.1-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:5dff14e75cdbcf07cdaa1c7707db6017d130f0af9ac41f6ce443a93318d6c6e0", size = 42938505 }, - { url = "https://files.pythonhosted.org/packages/d6/07/427859116bdd71847c898180f01802691f203c3e2455a1eb496130ff07c5/scipy-1.15.1-cp310-cp310-win_amd64.whl", hash = "sha256:f82fcf4e5b377f819542fbc8541f7b5fbcf1c0017d0df0bc22c781bf60abc4d8", size = 43909663 }, - { url = "https://files.pythonhosted.org/packages/8e/2e/7b71312da9c2dabff53e7c9a9d08231bc34d9d8fdabe88a6f1155b44591c/scipy-1.15.1-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:5bd8d27d44e2c13d0c1124e6a556454f52cd3f704742985f6b09e75e163d20d2", size = 41424362 }, - { url = "https://files.pythonhosted.org/packages/81/8c/ab85f1aa1cc200c796532a385b6ebf6a81089747adc1da7482a062acc46c/scipy-1.15.1-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:be3deeb32844c27599347faa077b359584ba96664c5c79d71a354b80a0ad0ce0", size = 32535910 }, - { url = "https://files.pythonhosted.org/packages/3b/9c/6f4b787058daa8d8da21ddff881b4320e28de4704a65ec147adb50cb2230/scipy-1.15.1-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:5eb0ca35d4b08e95da99a9f9c400dc9f6c21c424298a0ba876fdc69c7afacedf", size = 24809398 }, - { url = "https://files.pythonhosted.org/packages/16/2b/949460a796df75fc7a1ee1becea202cf072edbe325ebe29f6d2029947aa7/scipy-1.15.1-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:74bb864ff7640dea310a1377d8567dc2cb7599c26a79ca852fc184cc851954ac", size = 27918045 }, - { url = "https://files.pythonhosted.org/packages/5f/36/67fe249dd7ccfcd2a38b25a640e3af7e59d9169c802478b6035ba91dfd6d/scipy-1.15.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:667f950bf8b7c3a23b4199db24cb9bf7512e27e86d0e3813f015b74ec2c6e3df", size = 38332074 }, - { url = "https://files.pythonhosted.org/packages/fc/da/452e1119e6f720df3feb588cce3c42c5e3d628d4bfd4aec097bd30b7de0c/scipy-1.15.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:395be70220d1189756068b3173853029a013d8c8dd5fd3d1361d505b2aa58fa7", size = 40588469 }, - { url = "https://files.pythonhosted.org/packages/7f/71/5f94aceeac99a4941478af94fe9f459c6752d497035b6b0761a700f5f9ff/scipy-1.15.1-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:ce3a000cd28b4430426db2ca44d96636f701ed12e2b3ca1f2b1dd7abdd84b39a", size = 42965214 }, - { url = "https://files.pythonhosted.org/packages/af/25/caa430865749d504271757cafd24066d596217e83326155993980bc22f97/scipy-1.15.1-cp311-cp311-win_amd64.whl", hash = "sha256:3fe1d95944f9cf6ba77aa28b82dd6bb2a5b52f2026beb39ecf05304b8392864b", size = 43896034 }, - { url = "https://files.pythonhosted.org/packages/d8/6e/a9c42d0d39e09ed7fd203d0ac17adfea759cba61ab457671fe66e523dbec/scipy-1.15.1-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:c09aa9d90f3500ea4c9b393ee96f96b0ccb27f2f350d09a47f533293c78ea776", size = 41478318 }, - { url = "https://files.pythonhosted.org/packages/04/ee/e3e535c81828618878a7433992fecc92fa4df79393f31a8fea1d05615091/scipy-1.15.1-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:0ac102ce99934b162914b1e4a6b94ca7da0f4058b6d6fd65b0cef330c0f3346f", size = 32596696 }, - { url = "https://files.pythonhosted.org/packages/c4/5e/b1b0124be8e76f87115f16b8915003eec4b7060298117715baf13f51942c/scipy-1.15.1-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:09c52320c42d7f5c7748b69e9f0389266fd4f82cf34c38485c14ee976cb8cb04", size = 24870366 }, - { url = "https://files.pythonhosted.org/packages/14/36/c00cb73eefda85946172c27913ab995c6ad4eee00fa4f007572e8c50cd51/scipy-1.15.1-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:cdde8414154054763b42b74fe8ce89d7f3d17a7ac5dd77204f0e142cdc9239e9", size = 28007461 }, - { url = "https://files.pythonhosted.org/packages/68/94/aff5c51b3799349a9d1e67a056772a0f8a47db371e83b498d43467806557/scipy-1.15.1-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4c9d8fc81d6a3b6844235e6fd175ee1d4c060163905a2becce8e74cb0d7554ce", size = 38068174 }, - { url = "https://files.pythonhosted.org/packages/b0/3c/0de11ca154e24a57b579fb648151d901326d3102115bc4f9a7a86526ce54/scipy-1.15.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0fb57b30f0017d4afa5fe5f5b150b8f807618819287c21cbe51130de7ccdaed2", size = 40249869 }, - { url = "https://files.pythonhosted.org/packages/15/09/472e8d0a6b33199d1bb95e49bedcabc0976c3724edd9b0ef7602ccacf41e/scipy-1.15.1-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:491d57fe89927fa1aafbe260f4cfa5ffa20ab9f1435025045a5315006a91b8f5", size = 42629068 }, - { url = "https://files.pythonhosted.org/packages/ff/ba/31c7a8131152822b3a2cdeba76398ffb404d81d640de98287d236da90c49/scipy-1.15.1-cp312-cp312-win_amd64.whl", hash = "sha256:900f3fa3db87257510f011c292a5779eb627043dd89731b9c461cd16ef76ab3d", size = 43621992 }, - { url = "https://files.pythonhosted.org/packages/2b/bf/dd68965a4c5138a630eeed0baec9ae96e5d598887835bdde96cdd2fe4780/scipy-1.15.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:100193bb72fbff37dbd0bf14322314fc7cbe08b7ff3137f11a34d06dc0ee6b85", size = 41441136 }, - { url = "https://files.pythonhosted.org/packages/ef/5e/4928581312922d7e4d416d74c416a660addec4dd5ea185401df2269ba5a0/scipy-1.15.1-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:2114a08daec64980e4b4cbdf5bee90935af66d750146b1d2feb0d3ac30613692", size = 32533699 }, - { url = "https://files.pythonhosted.org/packages/32/90/03f99c43041852837686898c66767787cd41c5843d7a1509c39ffef683e9/scipy-1.15.1-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:6b3e71893c6687fc5e29208d518900c24ea372a862854c9888368c0b267387ab", size = 24807289 }, - { url = "https://files.pythonhosted.org/packages/9d/52/bfe82b42ae112eaba1af2f3e556275b8727d55ac6e4932e7aef337a9d9d4/scipy-1.15.1-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:837299eec3d19b7e042923448d17d95a86e43941104d33f00da7e31a0f715d3c", size = 27929844 }, - { url = "https://files.pythonhosted.org/packages/f6/77/54ff610bad600462c313326acdb035783accc6a3d5f566d22757ad297564/scipy-1.15.1-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82add84e8a9fb12af5c2c1a3a3f1cb51849d27a580cb9e6bd66226195142be6e", size = 38031272 }, - { url = "https://files.pythonhosted.org/packages/f1/26/98585cbf04c7cf503d7eb0a1966df8a268154b5d923c5fe0c1ed13154c49/scipy-1.15.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:070d10654f0cb6abd295bc96c12656f948e623ec5f9a4eab0ddb1466c000716e", size = 40210217 }, - { url = "https://files.pythonhosted.org/packages/fd/3f/3d2285eb6fece8bc5dbb2f9f94d61157d61d155e854fd5fea825b8218f12/scipy-1.15.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:55cc79ce4085c702ac31e49b1e69b27ef41111f22beafb9b49fea67142b696c4", size = 42587785 }, - { url = "https://files.pythonhosted.org/packages/48/7d/5b5251984bf0160d6533695a74a5fddb1fa36edd6f26ffa8c871fbd4782a/scipy-1.15.1-cp313-cp313-win_amd64.whl", hash = "sha256:c352c1b6d7cac452534517e022f8f7b8d139cd9f27e6fbd9f3cbd0bfd39f5bef", size = 43640439 }, - { url = "https://files.pythonhosted.org/packages/e7/b8/0e092f592d280496de52e152582030f8a270b194f87f890e1a97c5599b81/scipy-1.15.1-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:0458839c9f873062db69a03de9a9765ae2e694352c76a16be44f93ea45c28d2b", size = 41619862 }, - { url = "https://files.pythonhosted.org/packages/f6/19/0b6e1173aba4db9e0b7aa27fe45019857fb90d6904038b83927cbe0a6c1d/scipy-1.15.1-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:af0b61c1de46d0565b4b39c6417373304c1d4f5220004058bdad3061c9fa8a95", size = 32610387 }, - { url = "https://files.pythonhosted.org/packages/e7/02/754aae3bd1fa0f2479ade3cfdf1732ecd6b05853f63eee6066a32684563a/scipy-1.15.1-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:71ba9a76c2390eca6e359be81a3e879614af3a71dfdabb96d1d7ab33da6f2364", size = 24883814 }, - { url = "https://files.pythonhosted.org/packages/1f/ac/d7906201604a2ea3b143bb0de51b3966f66441ba50b7dc182c4505b3edf9/scipy-1.15.1-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:14eaa373c89eaf553be73c3affb11ec6c37493b7eaaf31cf9ac5dffae700c2e0", size = 27944865 }, - { url = "https://files.pythonhosted.org/packages/84/9d/8f539002b5e203723af6a6f513a45e0a7671e9dabeedb08f417ac17e4edc/scipy-1.15.1-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f735bc41bd1c792c96bc426dece66c8723283695f02df61dcc4d0a707a42fc54", size = 39883261 }, - { url = "https://files.pythonhosted.org/packages/97/c0/62fd3bab828bcccc9b864c5997645a3b86372a35941cdaf677565c25c98d/scipy-1.15.1-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:2722a021a7929d21168830790202a75dbb20b468a8133c74a2c0230c72626b6c", size = 42093299 }, - { url = "https://files.pythonhosted.org/packages/e4/1f/5d46a8d94e9f6d2c913cbb109e57e7eed914de38ea99e2c4d69a9fc93140/scipy-1.15.1-cp313-cp313t-win_amd64.whl", hash = "sha256:bc7136626261ac1ed988dca56cfc4ab5180f75e0ee52e58f1e6aa74b5f3eacd5", size = 43181730 }, +sdist = { url = "https://files.pythonhosted.org/packages/d9/7b/2b8ac283cf32465ed08bc20a83d559fe7b174a484781702ba8accea001d6/scipy-1.15.0.tar.gz", hash = "sha256:300742e2cc94e36a2880ebe464a1c8b4352a7b0f3e36ec3d2ac006cdbe0219ac", size = 59407226 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/6a/14ce8d4452acdced1b69ea32b0d304b04b00376deb4f1eb65f946aee41af/scipy-1.15.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:aeac60d3562a7bf2f35549bdfdb6b1751c50590f55ce7322b4b2fc821dc27fca", size = 41413763 }, + { url = "https://files.pythonhosted.org/packages/45/12/570ba186d0ae1d528f8f0524b88fb9a263653ce575ac085edd9c1ef29e9c/scipy-1.15.0-cp310-cp310-macosx_12_0_arm64.whl", hash = "sha256:5abbdc6ede5c5fed7910cf406a948e2c0869231c0db091593a6b2fa78be77e5d", size = 32518980 }, + { url = "https://files.pythonhosted.org/packages/51/5a/b6ac5aa213cfa196d15db5ee159010aa9b94d0bc2bfa917fb99297701628/scipy-1.15.0-cp310-cp310-macosx_14_0_arm64.whl", hash = "sha256:eb1533c59f0ec6c55871206f15a5c72d1fae7ad3c0a8ca33ca88f7c309bbbf8c", size = 24792491 }, + { url = "https://files.pythonhosted.org/packages/35/1f/6af575b77b2ee057551643de75a30252ce32098b2d9fd45bcf969a6fa35b/scipy-1.15.0-cp310-cp310-macosx_14_0_x86_64.whl", hash = "sha256:de112c2dae53107cfeaf65101419662ac0a54e9a088c17958b51c95dac5de56d", size = 27886039 }, + { url = "https://files.pythonhosted.org/packages/6a/7b/0c261d4857f459de6dffe11b3818583944f8d87716ce0b3b5f058aa34ff3/scipy-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2240e1fd0782e62e1aacdc7234212ee271d810f67e9cd3b8d521003a82603ef8", size = 38374628 }, + { url = "https://files.pythonhosted.org/packages/99/17/ca390fbbfea5b34e3a00fc819fcb7c22e8b889360882820030b533d26c01/scipy-1.15.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d35aef233b098e4de88b1eac29f0df378278e7e250a915766786b773309137c4", size = 40599127 }, + { url = "https://files.pythonhosted.org/packages/1d/65/95d93b1360f5defc1b6bf0963ac4e0d3413c95d8e8d6a1624a256506dfd3/scipy-1.15.0-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:1b29e4fc02e155a5fd1165f1e6a73edfdd110470736b0f48bcbe48083f0eee37", size = 42937900 }, + { url = "https://files.pythonhosted.org/packages/51/8c/c2d371111961f737ae08881f654cf54eca796c42ec0429add2a06df97049/scipy-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:0e5b34f8894f9904cc578008d1a9467829c1817e9f9cb45e6d6eeb61d2ab7731", size = 43907603 }, + { url = "https://files.pythonhosted.org/packages/b8/53/7f627c180cdaa211fa537650ca05912f58cb68fc33bb2f9af3d29169913e/scipy-1.15.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:46e91b5b16909ff79224b56e19cbad65ca500b3afda69225820aa3afbf9ec020", size = 41423594 }, + { url = "https://files.pythonhosted.org/packages/c9/ab/f848933c6f656f2c7af2d56d0be44511b730498538fe04db70eb03a6ad86/scipy-1.15.0-cp311-cp311-macosx_12_0_arm64.whl", hash = "sha256:82bff2eb01ccf7cea8b6ee5274c2dbeadfdac97919da308ee6d8e5bcbe846443", size = 32535797 }, + { url = "https://files.pythonhosted.org/packages/41/93/266693c471ec1e2e7748c1ee5e867299f3d0ac42e0e63f52649430ec1976/scipy-1.15.0-cp311-cp311-macosx_14_0_arm64.whl", hash = "sha256:9c8254fe21dd2c6c8f7757035ec0c31daecf3bb3cffd93bc1ca661b731d28136", size = 24809325 }, + { url = "https://files.pythonhosted.org/packages/f3/55/1acc49a48bc11fb95cf625c0763f2749f8710265d2fecbf6ed6dd618fc54/scipy-1.15.0-cp311-cp311-macosx_14_0_x86_64.whl", hash = "sha256:c9624eeae79b18cab1a31944b5ef87aa14b125d6ab69b71db22f0dbd962caf1e", size = 27917711 }, + { url = "https://files.pythonhosted.org/packages/e2/f5/15f62812b36f2f94b9d1ca31d3d2bbabfb6979e48a0866041bee7031c461/scipy-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d13bbc0658c11f3d19df4138336e4bce2c4fbd78c2755be4bf7b8e235481557f", size = 38331850 }, + { url = "https://files.pythonhosted.org/packages/ad/21/6dc57f6f6c8014dc6d07111e4976422580789fa96c4d7ddf63614939cb6c/scipy-1.15.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bdca4c7bb8dc41307e5f39e9e5d19c707d8e20a29845e7533b3bb20a9d4ccba0", size = 40587953 }, + { url = "https://files.pythonhosted.org/packages/da/dd/26db78c2054f8d81b28ae4688da7930ea3c33e5d1885928aadefeec979f9/scipy-1.15.0-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:6f376d7c767731477bac25a85d0118efdc94a572c6b60decb1ee48bf2391a73b", size = 42963920 }, + { url = "https://files.pythonhosted.org/packages/82/89/eb4aaf929be0e2c03bb5e40ed61427aab9c8ba6c0764aebf82d7302bb3d3/scipy-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:61513b989ee8d5218fbeb178b2d51534ecaddba050db949ae99eeb3d12f6825d", size = 43894857 }, + { url = "https://files.pythonhosted.org/packages/35/70/fffb90a725dec6056c9059073856fd99de22a253459a874a63b8b8a012db/scipy-1.15.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5beb0a2200372b7416ec73fdae94fe81a6e85e44eb49c35a11ac356d2b8eccc6", size = 41475240 }, + { url = "https://files.pythonhosted.org/packages/63/ca/6b838a2e5e6718d879e8522d1155a068c2a769be04f7da8c5179ead32a7b/scipy-1.15.0-cp312-cp312-macosx_12_0_arm64.whl", hash = "sha256:fde0f3104dfa1dfbc1f230f65506532d0558d43188789eaf68f97e106249a913", size = 32595923 }, + { url = "https://files.pythonhosted.org/packages/b1/07/4e69f6f7185915d77719bf226c1d554a4bb99f27cb92161fdd57b1434343/scipy-1.15.0-cp312-cp312-macosx_14_0_arm64.whl", hash = "sha256:35c68f7044b4e7ad73a3e68e513dda946989e523df9b062bd3cf401a1a882192", size = 24869617 }, + { url = "https://files.pythonhosted.org/packages/30/22/e3dadf189dcab215be461efe0fd9d288f4c2d99783c4aec2ce80837800b7/scipy-1.15.0-cp312-cp312-macosx_14_0_x86_64.whl", hash = "sha256:52475011be29dfcbecc3dfe3060e471ac5155d72e9233e8d5616b84e2b542054", size = 28007674 }, + { url = "https://files.pythonhosted.org/packages/51/0f/71c9ee2acaac0660a79e36424d367ed5737e4ef27b885f96cd439f451467/scipy-1.15.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5972e3f96f7dda4fd3bb85906a17338e65eaddfe47f750e240f22b331c08858e", size = 38066684 }, + { url = "https://files.pythonhosted.org/packages/fb/77/74a1ceecb205f5d46fe2cd10071383748ee8891a96b7824a372391a6291c/scipy-1.15.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fe00169cf875bed0b3c40e4da45b57037dc21d7c7bf0c85ed75f210c281488f1", size = 40250011 }, + { url = "https://files.pythonhosted.org/packages/8c/9f/f1544110a3d31183034e05422836505beb438aa56183f2ccef6dcd3b4e3f/scipy-1.15.0-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:161f80a98047c219c257bf5ce1777c574bde36b9d962a46b20d0d7e531f86863", size = 42625471 }, + { url = "https://files.pythonhosted.org/packages/3f/39/a29b75f9c30084cbafd416bfa00933311a5b7a96be6e88750c98521d2ccb/scipy-1.15.0-cp312-cp312-win_amd64.whl", hash = "sha256:327163ad73e54541a675240708244644294cb0a65cca420c9c79baeb9648e479", size = 43622832 }, + { url = "https://files.pythonhosted.org/packages/4d/46/2fa07d5b53092b73c4bb416954d07d883b53be4a5bd6282c67e03c051225/scipy-1.15.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:0fcb16eb04d84670722ce8d93b05257df471704c913cb0ff9dc5a1c31d1e9422", size = 41438080 }, + { url = "https://files.pythonhosted.org/packages/55/05/77778b1127e170ffb484614691fdd8f9d2640dcf951d515f513debe5d0e0/scipy-1.15.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:767e8cf6562931f8312f4faa7ddea412cb783d8df49e62c44d00d89f41f9bbe8", size = 32532932 }, + { url = "https://files.pythonhosted.org/packages/2b/9f/6de4970a2f524785d94a85f423a53b8c53d84917f2df702733ccdc9afd54/scipy-1.15.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:37ce9394cdcd7c5f437583fc6ef91bd290014993900643fdfc7af9b052d1613b", size = 24806488 }, + { url = "https://files.pythonhosted.org/packages/65/ef/b1c1e2499189bbea109a6b022a6147dd4552d72bed19289b4d4e411c4ce7/scipy-1.15.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:6d26f17c64abd6c6c2dfb39920f61518cc9e213d034b45b2380e32ba78fde4c0", size = 27930055 }, + { url = "https://files.pythonhosted.org/packages/24/ec/6e4fe2a34a91102c806ecf9f45426f66bd604a5b5f48e951ce2bd770b2fe/scipy-1.15.0-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1e2448acd79c6374583581a1ded32ac71a00c2b9c62dfa87a40e1dd2520be111", size = 38031212 }, + { url = "https://files.pythonhosted.org/packages/82/4d/ecef655956ce332edbc411ab64ab843d767dd86e646898ac721dbcc7910e/scipy-1.15.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36be480e512d38db67f377add5b759fb117edd987f4791cdf58e59b26962bee4", size = 40209536 }, + { url = "https://files.pythonhosted.org/packages/c5/ec/3af823fcd86e3155ad7ed2b684634391e4524ff82735c26abed522fc5405/scipy-1.15.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:ccb6248a9987193fe74363a2d73b93bc2c546e0728bd786050b7aef6e17db03c", size = 42584473 }, + { url = "https://files.pythonhosted.org/packages/23/01/f0ec4236ba8a96353e56694160041d7d9bebd9a0231a1c9beedc6e75cd50/scipy-1.15.0-cp313-cp313-win_amd64.whl", hash = "sha256:952d2e9eaa787f0a9e95b6e85da3654791b57a156c3e6609e65cc5176ccfe6f2", size = 43639460 }, + { url = "https://files.pythonhosted.org/packages/e9/02/c8bccc5c4813eccfeeef6ed0effe42e2cf98199d350ca476c22029569edc/scipy-1.15.0-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:b1432102254b6dc7766d081fa92df87832ac25ff0b3d3a940f37276e63eb74ff", size = 41642304 }, + { url = "https://files.pythonhosted.org/packages/27/7a/9191a8b61f5826f08932b6ae47d44fbf4f473beb307d8ca3ed96a216929f/scipy-1.15.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:4e08c6a36f46abaedf765dd2dfcd3698fa4bd7e311a9abb2d80e33d9b2d72c34", size = 32620019 }, + { url = "https://files.pythonhosted.org/packages/e6/17/9c8452c8a59f1ede4a7ba6ba03b8b44703cdd1f1217b649f470c216f3095/scipy-1.15.0-cp313-cp313t-macosx_14_0_arm64.whl", hash = "sha256:ec915cd26d76f6fc7ae8522f74f5b2accf39546f341c771bb2297f3871934a52", size = 24893299 }, + { url = "https://files.pythonhosted.org/packages/db/73/45c8566538bf9252be1e3e36b149714619c6f4d015a901cd76e257f88a37/scipy-1.15.0-cp313-cp313t-macosx_14_0_x86_64.whl", hash = "sha256:351899dd2a801edd3691622172bc8ea01064b1cada794f8641b89a7dc5418db6", size = 27955764 }, + { url = "https://files.pythonhosted.org/packages/9f/4e/8822a2cafcea8727430e9a0bf785e8f0e81aaaac1048dad764d522f0f1ec/scipy-1.15.0-cp313-cp313t-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9baff912ea4f78a543d183ed6f5b3bea9784509b948227daaf6f10727a0e2e5", size = 39879164 }, + { url = "https://files.pythonhosted.org/packages/b1/27/b55549a4aba515d9a19b6384c2c2f976725cd19d5d41c58ffac9a4d98892/scipy-1.15.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:cd9d9198a7fd9a77f0eb5105ea9734df26f41faeb2a88a0e62e5245506f7b6df", size = 42091406 }, + { url = "https://files.pythonhosted.org/packages/79/df/989b2fd3f8ead6bcf89fc683fde94741eb3b291e41a3ce70cec08c80aa36/scipy-1.15.0-cp313-cp313t-win_amd64.whl", hash = "sha256:129f899ed275c0515d553b8d31696924e2ca87d1972421e46c376b9eb87de3d2", size = 43188844 }, ] [[package]] @@ -4780,6 +4810,12 @@ ollama = [ onnx = [ { name = "onnxruntime-genai", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] +openai-realtime = [ + { name = "openai", extra = ["realtime"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pyaudio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pydub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "sounddevice", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] pandas = [ { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] @@ -4851,6 +4887,7 @@ requires-dist = [ { name = "ollama", marker = "extra == 'ollama'", specifier = "~=0.4" }, { name = "onnxruntime-genai", marker = "extra == 'onnx'", specifier = "~=0.5" }, { name = "openai", specifier = "~=1.0" }, + { name = "openai", extras = ["realtime"], marker = "extra == 'openai-realtime'", specifier = "~=1.0" }, { name = "openapi-core", specifier = ">=0.18,<0.20" }, { name = "opentelemetry-api", specifier = "~=1.24" }, { name = "opentelemetry-sdk", specifier = "~=1.24" }, @@ -4862,12 +4899,14 @@ requires-dist = [ { name = "pybars4", specifier = "~=0.9" }, { name = "pydantic", specifier = ">=2.0,!=2.10.0,!=2.10.1,!=2.10.2,!=2.10.3,<2.11" }, { name = "pydantic-settings", specifier = "~=2.0" }, + { name = "pydub", marker = "extra == 'openai-realtime'" }, { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=2.3,<2.6" }, { name = "pymongo", marker = "extra == 'mongo'", specifier = ">=4.8.0,<4.11" }, { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = "~=1.9" }, { name = "redis", extras = ["hiredis"], marker = "extra == 'redis'", specifier = "~=5.0" }, { name = "redisvl", marker = "extra == 'redis'", specifier = ">=0.3.6" }, { name = "sentence-transformers", marker = "extra == 'hugging-face'", specifier = ">=2.2,<4.0" }, + { name = "sounddevice", marker = "extra == 'openai-realtime'" }, { name = "torch", marker = "extra == 'hugging-face'", specifier = "==2.5.1" }, { name = "transformers", extras = ["torch"], marker = "extra == 'hugging-face'", specifier = "~=4.28" }, { name = "types-redis", marker = "extra == 'redis'", specifier = "~=4.6.0.20240425" }, @@ -4911,11 +4950,11 @@ wheels = [ [[package]] name = "setuptools" -version = "75.8.0" +version = "75.7.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/92/ec/089608b791d210aec4e7f97488e67ab0d33add3efccb83a056cbafe3a2a6/setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", size = 1343222 } +sdist = { url = "https://files.pythonhosted.org/packages/ac/57/e6f0bde5a2c333a32fbcce201f906c1fd0b3a7144138712a5e9d9598c5ec/setuptools-75.7.0.tar.gz", hash = "sha256:886ff7b16cd342f1d1defc16fc98c9ce3fde69e087a4e1983d7ab634e5f41f4f", size = 1338616 } wheels = [ - { url = "https://files.pythonhosted.org/packages/69/8a/b9dc7678803429e4a3bc9ba462fa3dd9066824d3c607490235c6a796be5a/setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3", size = 1228782 }, + { url = "https://files.pythonhosted.org/packages/4e/6e/abdfaaf5c294c553e7a81cf5d801fbb4f53f5c5b6646de651f92a2667547/setuptools-75.7.0-py3-none-any.whl", hash = "sha256:84fb203f278ebcf5cd08f97d3fb96d3fbed4b629d500b29ad60d11e00769b183", size = 1224467 }, ] [[package]] @@ -5071,6 +5110,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/93/84a16940c44f6ec62cf334f25aed3128a514dffc361397eee09421a1c7f2/snoop-0.6.0-py3-none-any.whl", hash = "sha256:f5ea9060e65594bf404e6841086b4a964cc27bc30569109c91a470f948b0f729", size = 27461 }, ] +[[package]] +name = "sounddevice" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/2d/b04ae180312b81dbb694504bee170eada5372242e186f6298139fd3a0513/sounddevice-0.5.1.tar.gz", hash = "sha256:09ca991daeda8ce4be9ac91e15a9a81c8f81efa6b695a348c9171ea0c16cb041", size = 52896 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/d1/464b5fca3decdd0cfec8c47f7b4161a0b12972453201c1bf03811f367c5e/sounddevice-0.5.1-py3-none-any.whl", hash = "sha256:e2017f182888c3f3c280d9fbac92e5dbddac024a7e3442f6e6116bd79dab8a9c", size = 32276 }, + { url = "https://files.pythonhosted.org/packages/6f/f6/6703fe7cf3d7b7279040c792aeec6334e7305956aba4a80f23e62c8fdc44/sounddevice-0.5.1-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:d16cb23d92322526a86a9490c427bf8d49e273d9ccc0bd096feecd229cde6031", size = 107916 }, + { url = "https://files.pythonhosted.org/packages/57/a5/78a5e71f5ec0faedc54f4053775d61407bfbd7d0c18228c7f3d4252fd276/sounddevice-0.5.1-py3-none-win32.whl", hash = "sha256:d84cc6231526e7a08e89beff229c37f762baefe5e0cc2747cbe8e3a565470055", size = 312494 }, + { url = "https://files.pythonhosted.org/packages/af/9b/15217b04f3b36d30de55fef542389d722de63f1ad81f9c72d8afc98cb6ab/sounddevice-0.5.1-py3-none-win_amd64.whl", hash = "sha256:4313b63f2076552b23ac3e0abd3bcfc0c1c6a696fc356759a13bd113c9df90f1", size = 363634 }, +] + [[package]] name = "soupsieve" version = "2.6" @@ -5406,11 +5460,11 @@ wheels = [ [[package]] name = "types-setuptools" -version = "75.8.0.20250110" +version = "75.6.0.20241223" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f7/42/5713e90d4f9683f2301d900f33e4fc2405ad8ac224dda30f6cb7f4cd215b/types_setuptools-75.8.0.20250110.tar.gz", hash = "sha256:96f7ec8bbd6e0a54ea180d66ad68ad7a1d7954e7281a710ea2de75e355545271", size = 48185 } +sdist = { url = "https://files.pythonhosted.org/packages/53/48/a89068ef20e3bbb559457faf0fd3c18df6df5df73b4b48ebf466974e1f54/types_setuptools-75.6.0.20241223.tar.gz", hash = "sha256:d9478a985057ed48a994c707f548e55aababa85fe1c9b212f43ab5a1fffd3211", size = 48063 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/a3/dbfd106751b11c728cec21cc62cbfe7ff7391b935c4b6e8f0bdc2e6fd541/types_setuptools-75.8.0.20250110-py3-none-any.whl", hash = "sha256:a9f12980bbf9bcdc23ecd80755789085bad6bfce4060c2275bc2b4ca9f2bc480", size = 71521 }, + { url = "https://files.pythonhosted.org/packages/41/2f/051d5d23711209d4077d95c62fa8ef6119df7298635e3a929e50376219d1/types_setuptools-75.6.0.20241223-py3-none-any.whl", hash = "sha256:7cbfd3bf2944f88bbcdd321b86ddd878232a277be95d44c78a53585d78ebc2f6", size = 71377 }, ] [[package]] @@ -5632,16 +5686,16 @@ wheels = [ [[package]] name = "virtualenv" -version = "20.29.0" +version = "20.28.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "distlib", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "platformdirs", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/5a/5d/8d625ebddf9d31c301f85125b78002d4e4401fe1c15c04dca58a54a3056a/virtualenv-20.29.0.tar.gz", hash = "sha256:6345e1ff19d4b1296954cee076baaf58ff2a12a84a338c62b02eda39f20aa982", size = 7658081 } +sdist = { url = "https://files.pythonhosted.org/packages/50/39/689abee4adc85aad2af8174bb195a819d0be064bf55fcc73b49d2b28ae77/virtualenv-20.28.1.tar.gz", hash = "sha256:5d34ab240fdb5d21549b76f9e8ff3af28252f5499fb6d6f031adac4e5a8c5329", size = 7650532 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/d3/12687ab375bb0e077ea802a5128f7b45eb5de7a7c6cb576ccf9dd59ff80a/virtualenv-20.29.0-py3-none-any.whl", hash = "sha256:c12311863497992dc4b8644f8ea82d3b35bb7ef8ee82e6630d76d0197c39baf9", size = 4282443 }, + { url = "https://files.pythonhosted.org/packages/51/8f/dfb257ca6b4e27cb990f1631142361e4712badab8e3ca8dc134d96111515/virtualenv-20.28.1-py3-none-any.whl", hash = "sha256:412773c85d4dab0409b83ec36f7a6499e72eaf08c80e81e9576bca61831c71cb", size = 4276719 }, ] [[package]] @@ -5720,7 +5774,7 @@ wheels = [ [[package]] name = "weaviate-client" -version = "4.10.4" +version = "4.10.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "authlib", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -5731,9 +5785,9 @@ dependencies = [ { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "validators", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/ce/e34426eeda39a77b45df86f9ab901a7232096a071ee379a046a8072e2a35/weaviate_client-4.10.4.tar.gz", hash = "sha256:a1e799fc41d9f43a56c95490f6c14f475861f27d2a62b9b6de28a1db5494751d", size = 594549 } +sdist = { url = "https://files.pythonhosted.org/packages/e8/80/5e36a1923d0bc01a6151f1cfb1550da83efec340cded1c4f885615e09575/weaviate_client-4.10.2.tar.gz", hash = "sha256:fde5ad8e36604674d26b115288b58a7e182c91e36c2b41a00d18a36fe4ec7e3f", size = 587835 } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/e9/5b6ffbdee0d0f1444d0ce142c70a70bf22ba43bf2d6b35913a8d7e674431/weaviate_client-4.10.4-py3-none-any.whl", hash = "sha256:d9808456ba109fcd99331bc833b61cf520bf6ad9db442db621e12f78c8480c4c", size = 330450 }, + { url = "https://files.pythonhosted.org/packages/80/ca/9f2f1f27a05bfe90cb35a6dacaa547ad5a133211aeca7bb0021e2bbabb06/weaviate_client-4.10.2-py3-none-any.whl", hash = "sha256:e1706438aa7b57be5443bbdebff206cc6688110d1669d54c2721b3aa640b2c4c", size = 325368 }, ] [[package]] From 9cd8c9e0990b0a7a0686ff04942ad3d1670b0023 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 9 Jan 2025 16:47:12 +0100 Subject: [PATCH 02/25] major update --- python/pyproject.toml | 5 +- .../audio/04-chat_with_realtime_api.py | 176 +++-- python/samples/concepts/audio/audio_player.py | 2 +- .../concepts/audio/audio_player_async.py | 4 +- .../concepts/audio/audio_recorder_stream.py | 3 +- .../ai/chat_completion_client_base.py | 12 - .../connectors/ai/function_calling_utils.py | 50 ++ .../connectors/ai/open_ai/__init__.py | 8 + .../open_ai/services/open_ai_realtime_base.py | 614 +++++++++++------- .../connectors/ai/realtime_client_base.py | 131 +++- .../tests/unit/contents/test_audio_content.py | 60 -- python/uv.lock | 460 +++++++------ 12 files changed, 891 insertions(+), 634 deletions(-) delete mode 100644 python/tests/unit/contents/test_audio_content.py diff --git a/python/pyproject.toml b/python/pyproject.toml index d6c7b2d42673..d828b1ec5aa9 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -124,10 +124,7 @@ dapr = [ "flask-dapr>=1.14.0" ] openai_realtime = [ - "openai[realtime] ~= 1.0", - "pyaudio", - "pydub", - "sounddevice" + "openai[realtime] ~= 1.0" ] [tool.uv] diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index 4440d13b8eec..bffbad691716 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -3,51 +3,60 @@ import contextlib import logging import signal +from typing import Any -from samples.concepts.audio.audio_player_async import AudioPlayerAsync +from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent -# This simple sample demonstrates how to use the OpenAI Realtime API to create -# a chat bot that can listen and respond directly through audio. -# It requires installing semantic-kernel[openai_realtime] which includes the -# OpenAI Realtime API client and some packages for handling audio locally. -# It has hardcoded device id's set in the AudioRecorderStream and AudioPlayerAsync classes, -# so you may need to adjust these for your system. +from samples.concepts.audio.audio_player_async import AudioPlayerAsync from samples.concepts.audio.audio_recorder_stream import AudioRecorderStream from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior -from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( +from semantic_kernel.connectors.ai.open_ai import ( + OpenAIRealtime, OpenAIRealtimeExecutionSettings, TurnDetection, ) -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.contents import AudioContent, ChatHistory, StreamingTextContent from semantic_kernel.functions import kernel_function logging.basicConfig(level=logging.WARNING) logger = logging.getLogger(__name__) +# This simple sample demonstrates how to use the OpenAI Realtime API to create +# a chat bot that can listen and respond directly through audio. +# It requires installing: +# - semantic-kernel[openai_realtime] +# - pyaudio +# - sounddevice +# - pydub +# e.g. pip install semantic-kernel[openai_realtime] pyaudio sounddevice pydub + +# The characterics of your speaker and microphone are a big factor in a smooth conversation +# so you may need to try out different devices for each. +# you can also play around with the turn_detection settings to get the best results. +# It has device id's set in the AudioRecorderStream and AudioPlayerAsync classes, +# so you may need to adjust these for your system. +# you can check the available devices by uncommenting line below the function + -def signal_handler(): - for task in asyncio.all_tasks(): - task.cancel() +def check_audio_devices(): + import sounddevice as sd # type: ignore + print(sd.query_devices()) -system_message = """ -You are a chat bot. Your name is Mosscap and -you have one goal: figure out what people need. -Your full name, should you need to know it, is -Splendid Speckled Mosscap. You communicate -effectively, but you tend to answer with long -flowery prose. -""" -history = ChatHistory() -history.add_user_message("Hi there, who are you?") -history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") +# check_audio_devices() class Speaker: - def __init__(self, audio_player: AudioPlayerAsync, realtime_client: OpenAIRealtime, kernel: Kernel): + """This is a simple class that opens the session with the realtime api and plays the audio response. + + At the same time it prints the transcript of the conversation to the console. + """ + + def __init__(self, audio_player: AudioPlayerAsync, realtime_client: RealtimeClientBase, kernel: Kernel): self.audio_player = audio_player self.realtime_client = realtime_client self.kernel = kernel @@ -56,42 +65,63 @@ async def play( self, chat_history: ChatHistory, settings: OpenAIRealtimeExecutionSettings, + print_transcript: bool = True, ) -> None: + # reset the frame count for the audio player self.audio_player.reset_frame_count() - print("Mosscap (transcript): ", end="") - try: - async for content in self.realtime_client.get_streaming_chat_message_content( - chat_history=chat_history, settings=settings, kernel=self.kernel - ): - if not content: - continue - for item in content.items: - match item: - case StreamingTextContent(): - print(item.text, end="") - await asyncio.sleep(0.01) - continue - case AudioContent(): - self.audio_player.add_data(item.data) - await asyncio.sleep(0.01) - continue - except asyncio.CancelledError: - print("\nThanks for talking to Mosscap!") + # open the connection to the realtime api + async with self.realtime_client as client: + # update the session with the chat_history and settings + await client.update_session(settings=settings, chat_history=chat_history) + # print the start message of the transcript + if print_transcript: + print("Mosscap (transcript): ", end="") + try: + # start listening for events + async for content in self.realtime_client.event_listener(settings=settings, kernel=self.kernel): + if not content: + continue + # the contents returned should be StreamingChatMessageContent + # so we will loop through the items within it. + for item in content.items: + match item: + case StreamingTextContent(): + if print_transcript: + print(item.text, end="") + await asyncio.sleep(0.01) + continue + case AudioContent(): + self.audio_player.add_data(item.data) + await asyncio.sleep(0.01) + continue + except asyncio.CancelledError: + print("\nThanks for talking to Mosscap!") class Microphone: - def __init__(self, audio_recorder: AudioRecorderStream, realtime_client: OpenAIRealtime): + """This is a simple class that opens the microphone and sends the audio to the realtime api.""" + + def __init__(self, audio_recorder: AudioRecorderStream, realtime_client: RealtimeClientBase): self.audio_recorder = audio_recorder self.realtime_client = realtime_client async def record_audio(self): with contextlib.suppress(asyncio.CancelledError): - async for audio in self.audio_recorder.stream_audio_content(): - if audio.data: - await self.realtime_client.send_content(content=audio) + async for content in self.audio_recorder.stream_audio_content(): + if content.data: + await self.realtime_client.send_event( + "input_audio_buffer.append", + content=content, + ) await asyncio.sleep(0.01) +# this function is used to stop the processes when ctrl + c is pressed +def signal_handler(): + for task in asyncio.all_tasks(): + task.cancel() + + @kernel_function def get_weather(location: str) -> str: """Get the weather for a location.""" @@ -99,23 +129,59 @@ def get_weather(location: str) -> str: return f"The weather in {location} is sunny." +def response_created_callback( + event: RealtimeServerEvent, settings: PromptExecutionSettings | None = None, **kwargs: Any +) -> None: + """Add a empty print to start a new line for a new response.""" + print("") + + async def main() -> None: + # setup the asyncio loop with the signal event handler loop = asyncio.get_event_loop() loop.add_signal_handler(signal.SIGINT, signal_handler) + + # create the Kernel and add a simple function for function calling. + kernel = Kernel() + kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) + + # create the realtime client and register the response created callback + realtime_client = OpenAIRealtime(ai_model_id="gpt-4o-realtime-preview-2024-12-17") + realtime_client.register_event_handler("response.created", response_created_callback) + + # create the speaker and microphone + speaker = Speaker(AudioPlayerAsync(device_id=7), realtime_client, kernel) + microphone = Microphone(AudioRecorderStream(device_id=2), realtime_client) + + # Create the settings for the session + # the key thing to decide on is to enable the server_vad turn detection + # if turn is turned off (by setting turn_detection=None), you will have to send + # the "input_audio_buffer.commit" and "response.create" event to the realtime api + # to signal the end of the user's turn and start the response. + + # The realtime api, does not use a system message, but takes instructions as a parameter for a session + instructions = """ + You are a chat bot. Your name is Mosscap and + you have one goal: figure out what people need. + Your full name, should you need to know it, is + Splendid Speckled Mosscap. You communicate + effectively, but you tend to answer with long + flowery prose. + """ + # but we can add a chat history to conversation after starting it + chat_history = ChatHistory() + chat_history.add_user_message("Hi there, who are you?") + chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") + settings = OpenAIRealtimeExecutionSettings( - instructions=system_message, + instructions=instructions, voice="sage", turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), function_choice_behavior=FunctionChoiceBehavior.Auto(), ) - realtime_client = OpenAIRealtime(ai_model_id="gpt-4o-realtime-preview-2024-12-17") - kernel = Kernel() - kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) - - speaker = Speaker(AudioPlayerAsync(), realtime_client, kernel) - microphone = Microphone(AudioRecorderStream(), realtime_client) + # start the the speaker and the microphone with contextlib.suppress(asyncio.CancelledError): - await asyncio.gather(*[speaker.play(history, settings), microphone.record_audio()]) + await asyncio.gather(*[speaker.play(chat_history, settings), microphone.record_audio()]) if __name__ == "__main__": diff --git a/python/samples/concepts/audio/audio_player.py b/python/samples/concepts/audio/audio_player.py index 036b978dcff1..b10c15184821 100644 --- a/python/samples/concepts/audio/audio_player.py +++ b/python/samples/concepts/audio/audio_player.py @@ -20,7 +20,7 @@ class AudioPlayer(BaseModel): # Audio replay parameters CHUNK: ClassVar[int] = 1024 - audio_content: AudioContent | None = None + audio_content: AudioContent def play(self, text: str | None = None) -> None: """Play the audio content to the default audio output device. diff --git a/python/samples/concepts/audio/audio_player_async.py b/python/samples/concepts/audio/audio_player_async.py index 9ae424b01c66..a77b8df6e32c 100644 --- a/python/samples/concepts/audio/audio_player_async.py +++ b/python/samples/concepts/audio/audio_player_async.py @@ -13,7 +13,7 @@ class AudioPlayerAsync: - def __init__(self): + def __init__(self, device_id: int | None = None): self.queue = [] self.lock = threading.Lock() self.stream = sd.OutputStream( @@ -22,7 +22,7 @@ def __init__(self): channels=CHANNELS, dtype=np.int16, blocksize=int(CHUNK_LENGTH_S * SAMPLE_RATE), - device=3, + device=device_id, ) self.playing = False self._frame_count = 0 diff --git a/python/samples/concepts/audio/audio_recorder_stream.py b/python/samples/concepts/audio/audio_recorder_stream.py index 99ac1a9f8141..55684e9c469b 100644 --- a/python/samples/concepts/audio/audio_recorder_stream.py +++ b/python/samples/concepts/audio/audio_recorder_stream.py @@ -28,6 +28,7 @@ class AudioRecorderStream(BaseModel): CHANNELS: ClassVar[int] = 1 SAMPLE_RATE: ClassVar[int] = 24000 CHUNK_LENGTH_S: ClassVar[float] = 0.05 + device_id: int | None = None async def stream_audio_content(self) -> AsyncGenerator[AudioContent, None]: import sounddevice as sd # type: ignore @@ -41,7 +42,7 @@ async def stream_audio_content(self) -> AsyncGenerator[AudioContent, None]: channels=self.CHANNELS, samplerate=self.SAMPLE_RATE, dtype="int16", - device=4, + device=self.device_id, ) stream.start() try: diff --git a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py index e24ae4954c42..c2fd0877ca09 100644 --- a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py @@ -438,16 +438,4 @@ def _yield_function_result_messages(self, function_result_messages: list) -> boo """ return len(function_result_messages) > 0 and len(function_result_messages[0].items) > 0 - async def _streaming_function_call_result_callback( - self, function_result_messages: list["ChatMessageContent"] - ) -> None: - """Callback to handle the streaming function call result messages. - - Override this method to handle the streaming function call result messages. - - Args: - function_result_messages (list): The streaming function call result messages. - """ - return - # endregion diff --git a/python/semantic_kernel/connectors/ai/function_calling_utils.py b/python/semantic_kernel/connectors/ai/function_calling_utils.py index 7a5c2950c4e0..1cad6d26ce6c 100644 --- a/python/semantic_kernel/connectors/ai/function_calling_utils.py +++ b/python/semantic_kernel/connectors/ai/function_calling_utils.py @@ -1,10 +1,13 @@ # Copyright (c) Microsoft. All rights reserved. from collections import OrderedDict +from collections.abc import Callable +from copy import deepcopy from typing import TYPE_CHECKING, Any from semantic_kernel.contents.utils.author_role import AuthorRole from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError +from semantic_kernel.utils.experimental_decorator import experimental_function if TYPE_CHECKING: from semantic_kernel.connectors.ai.function_choice_behavior import ( @@ -15,6 +18,7 @@ from semantic_kernel.contents.chat_message_content import ChatMessageContent from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent from semantic_kernel.functions.kernel_function_metadata import KernelFunctionMetadata + from semantic_kernel.kernel import Kernel def update_settings_from_function_call_configuration( @@ -134,3 +138,49 @@ def merge_streaming_function_results( function_invoke_attempt=function_invoke_attempt, ) ] + + +@experimental_function +def prepare_settings_for_function_calling( + settings: "PromptExecutionSettings", + settings_class: type["PromptExecutionSettings"], + update_settings_callback: Callable[..., None], + kernel: "Kernel", +) -> "PromptExecutionSettings": + """Prepare settings for the service. + + Args: + settings: Prompt execution settings. + settings_class: The settings class. + update_settings_callback: The callback to update the settings. + kernel: Kernel instance. + + Returns: + PromptExecutionSettings of type settings_class. + """ + settings = deepcopy(settings) + if not isinstance(settings, settings_class): + settings = settings_class.from_prompt_execution_settings(settings) + + # For backwards compatibility we need to convert the `FunctionCallBehavior` to `FunctionChoiceBehavior` + # if this method is called with a `FunctionCallBehavior` object as part of the settings + + from semantic_kernel.connectors.ai.function_call_behavior import FunctionCallBehavior + from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceBehavior + + if hasattr(settings, "function_call_behavior") and isinstance( + settings.function_call_behavior, FunctionCallBehavior + ): + settings.function_choice_behavior = FunctionChoiceBehavior.from_function_call_behavior( + settings.function_call_behavior + ) + + if settings.function_choice_behavior: + # Configure the function choice behavior into the settings object + # that will become part of the request to the AI service + settings.function_choice_behavior.configure( + kernel=kernel, + update_settings_callback=update_settings_callback, + settings=settings, + ) + return settings diff --git a/python/semantic_kernel/connectors/ai/open_ai/__init__.py b/python/semantic_kernel/connectors/ai/open_ai/__init__.py index a3103ae86446..27d36ea30d34 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/__init__.py +++ b/python/semantic_kernel/connectors/ai/open_ai/__init__.py @@ -22,6 +22,10 @@ OpenAIPromptExecutionSettings, OpenAITextPromptExecutionSettings, ) +from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( + OpenAIRealtimeExecutionSettings, + TurnDetection, +) from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_text_to_audio_execution_settings import ( OpenAITextToAudioExecutionSettings, ) @@ -36,6 +40,7 @@ from semantic_kernel.connectors.ai.open_ai.services.azure_text_to_image import AzureTextToImage from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text import OpenAIAudioToText from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletion +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding import OpenAITextEmbedding from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio import OpenAITextToAudio @@ -69,6 +74,8 @@ "OpenAIChatPromptExecutionSettings", "OpenAIEmbeddingPromptExecutionSettings", "OpenAIPromptExecutionSettings", + "OpenAIRealtime", + "OpenAIRealtimeExecutionSettings", "OpenAISettings", "OpenAITextCompletion", "OpenAITextEmbedding", @@ -77,4 +84,5 @@ "OpenAITextToAudioExecutionSettings", "OpenAITextToImage", "OpenAITextToImageExecutionSettings", + "TurnDetection", ] diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py index c73f12d7f343..4175d9449b2e 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -4,8 +4,10 @@ import base64 import logging import sys -from collections.abc import AsyncGenerator, Callable -from typing import TYPE_CHECKING, Any, ClassVar +from collections.abc import AsyncGenerator +from enum import Enum +from inspect import isawaitable +from typing import Any, ClassVar, Protocol, runtime_checkable if sys.version_info >= (3, 12): from typing import override # pragma: no cover @@ -15,19 +17,14 @@ from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection from openai.types.beta.realtime.conversation_item_create_event_param import ConversationItemParam from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent -from openai.types.beta.realtime.session import Session from pydantic import Field -from semantic_kernel.connectors.ai.chat_completion_client_base import ChatCompletionClientBase -from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration -from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType +from semantic_kernel.connectors.ai.function_calling_utils import prepare_settings_for_function_calling from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_utils import ( - update_settings_from_function_call_configuration, -) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.chat_history import ChatHistory from semantic_kernel.contents.function_call_content import FunctionCallContent from semantic_kernel.contents.function_result_content import FunctionResultContent from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent @@ -35,260 +32,401 @@ from semantic_kernel.contents.text_content import TextContent from semantic_kernel.contents.utils.author_role import AuthorRole from semantic_kernel.kernel import Kernel - -if TYPE_CHECKING: - from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.utils.experimental_decorator import experimental_class logger: logging.Logger = logging.getLogger(__name__) -class OpenAIRealtimeBase(OpenAIHandler, ChatCompletionClientBase): +@runtime_checkable +@experimental_class +class EventCallBackProtocolAsync(Protocol): + """Event callback protocol.""" + + async def __call__( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool] | None: + """Call the event callback.""" + ... + + +@runtime_checkable +@experimental_class +class EventCallBackProtocol(Protocol): + """Event callback protocol.""" + + def __call__( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool] | None: + """Call the event callback.""" + ... + + +@experimental_class +class SendEvents(str, Enum): + """Events that can be sent.""" + + SESSION_UPDATE = "session.update" + INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append" + INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit" + INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear" + CONVERSATION_ITEM_CREATE = "conversation.item.create" + CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate" + CONVERSATION_ITEM_DELETE = "conversation.item.delete" + RESPONSE_CREATE = "response.create" + RESPONSE_CANCEL = "response.cancel" + + +@experimental_class +class ListenEvents(str, Enum): + """Events that can be listened to.""" + + ERROR = "error" + SESSION_CREATED = "session.created" + SESSION_UPDATED = "session.updated" + CONVERSATION_CREATED = "conversation.created" + INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed" + INPUT_AUDIO_BUFFER_CLEARED = "input_audio_buffer.cleared" + INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started" + INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped" + CONVERSATION_ITEM_CREATED = "conversation.item.created" + CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_COMPLETED = "conversation.item.input_audio_transcription.completed" + CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_FAILED = "conversation.item.input_audio_transcription.failed" + CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated" + CONVERSATION_ITEM_DELETED = "conversation.item.deleted" + RESPONSE_CREATED = "response.created" + RESPONSE_DONE = "response.done" + RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added" + RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done" + RESPONSE_CONTENT_PART_ADDED = "response.content_part.added" + RESPONSE_CONTENT_PART_DONE = "response.content_part.done" + RESPONSE_TEXT_DELTA = "response.text.delta" + RESPONSE_TEXT_DONE = "response.text.done" + RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.audio_transcript.delta" + RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.audio_transcript.done" + RESPONSE_AUDIO_DELTA = "response.audio.delta" + RESPONSE_AUDIO_DONE = "response.audio.done" + RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" + RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" + RATE_LIMITS_UPDATED = "rate_limits.updated" + + +@experimental_class +class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): """OpenAI Realtime service.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True connection: AsyncRealtimeConnection | None = None connected: asyncio.Event = Field(default_factory=asyncio.Event) - session: Session | None = None + event_log: dict[str, list[RealtimeServerEvent]] = Field(default_factory=dict) + event_handlers: dict[str, list[EventCallBackProtocol | EventCallBackProtocolAsync]] = Field(default_factory=dict) - def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: - """Get the request settings class.""" - from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa - OpenAIRealtimeExecutionSettings, + def model_post_init(self, *args, **kwargs) -> None: + """Post init method for the model.""" + # Register the default event handlers + self.register_event_handler(ListenEvents.RESPONSE_AUDIO_DELTA, self.response_audio_delta_callback) + self.register_event_handler( + ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA, self.response_audio_transcript_delta_callback ) + self.register_event_handler( + ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE, self.response_audio_transcript_done_callback + ) + self.register_event_handler( + ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, self.response_function_call_arguments_delta_callback + ) + self.register_event_handler(ListenEvents.ERROR, self.error_callback) + self.register_event_handler(ListenEvents.SESSION_CREATED, self.session_callback) + self.register_event_handler(ListenEvents.SESSION_UPDATED, self.session_callback) - return OpenAIRealtimeExecutionSettings - - async def _get_connection(self) -> AsyncRealtimeConnection: - await self.connected.wait() - if not self.connection: - raise ValueError("Connection not established") - return self.connection + def register_event_handler( + self, event_type: str | ListenEvents, handler: EventCallBackProtocol | EventCallBackProtocolAsync + ) -> None: + """Register a event handler.""" + if not isinstance(event_type, ListenEvents): + event_type = ListenEvents(event_type) + self.event_handlers.setdefault(event_type, []).append(handler) @override - async def _inner_get_streaming_chat_message_contents( + async def event_listener( self, - chat_history: "ChatHistory", settings: "PromptExecutionSettings", - function_invoke_attempt: int = 0, + chat_history: "ChatHistory | None" = None, **kwargs: Any, - ) -> AsyncGenerator[list[StreamingChatMessageContent], Any]: - if not isinstance(settings, self.get_prompt_execution_settings_class()): - settings = self.get_prompt_execution_settings_from_settings(settings) - - events: list[RealtimeServerEvent] = [] - detailed_events: dict[str, list[RealtimeServerEvent]] = {} - function_calls: list[StreamingChatMessageContent] = [] - - async with self.client.beta.realtime.connect(model=self.ai_model_id) as conn: - self.connection = conn - self.connected.set() - - await conn.session.update(session=settings.prepare_settings_dict()) - if len(chat_history) > 0: - await asyncio.gather(*(self._add_content_to_conversation(msg) for msg in chat_history.messages)) - - async for event in conn: - events.append(event) - detailed_events.setdefault(event.type, []).append(event) - match event.type: - case "session.created" | "session.updated": - self.session = event.session - continue - case "error": - logger.error("Error received: %s", event.error) - continue - case "response.audio.delta": - yield [ - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[AudioContent(data=base64.b64decode(event.delta), data_format="base64")], - choice_index=event.content_index, - inner_content=event, - ) - ] + ) -> AsyncGenerator[StreamingChatMessageContent, Any]: + await self.connected.wait() + if not self.connection: + raise ValueError("Connection is not established.") + if not chat_history: + chat_history = ChatHistory() + async for event in self.connection: + event_type = ListenEvents(event.type) + self.event_log.setdefault(event_type, []).append(event) + for handler in self.event_handlers.get(event_type, []): + task = handler(event=event, settings=settings) + if not task: + continue + if isawaitable(task): + async_result = await task + if not async_result: continue - case "response.audio_transcript.delta": - yield [ - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[StreamingTextContent(text=event.delta, choice_index=event.content_index)], - choice_index=event.content_index, - inner_content=event, + result, should_return = async_result + else: + result, should_return = task + if should_return: + yield result + else: + chat_history.add_message(result) + + for event_type in self.event_log: + logger.debug(f"Event type: {event_type}, count: {len(self.event_log[event_type])}") + + @override + async def send_event(self, event: str | SendEvents, **kwargs: Any) -> None: + await self.connected.wait() + if not self.connection: + raise ValueError("Connection is not established.") + if not isinstance(event, SendEvents): + event = SendEvents(event) + match event: + case SendEvents.SESSION_UPDATE: + if "settings" not in kwargs: + logger.error("Event data does not contain 'settings'") + await self.connection.session.update(session=kwargs["settings"].prepare_settings_dict()) + case SendEvents.INPUT_AUDIO_BUFFER_APPEND: + if "content" not in kwargs: + logger.error("Event data does not contain 'content'") + return + await self.connection.input_audio_buffer.append(audio=kwargs["content"].data.decode("utf-8")) + case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: + await self.connection.input_audio_buffer.commit() + case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: + await self.connection.input_audio_buffer.clear() + case SendEvents.CONVERSATION_ITEM_CREATE: + if "item" not in kwargs: + logger.error("Event data does not contain 'item'") + return + content = kwargs["item"] + for item in content.items: + match item: + case TextContent(): + await self.connection.conversation.item.create( + item=ConversationItemParam( + type="message", + content=[ + { + "type": "input_text", + "text": item.text, + } + ], + role="user", + ) ) - ] - continue - case "response.audio_transcript.done": - chat_history.add_message( - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[StreamingTextContent(text=event.transcript, choice_index=event.content_index)], - choice_index=event.content_index, - inner_content=event, + case FunctionCallContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function call needs to have a call_id") + continue + await self.connection.conversation.item.create( + item=ConversationItemParam( + type="function_call", + name=item.name, + arguments=item.arguments, + call_id=call_id, + ) ) - ) - case "response.function_call_arguments.delta": - msg = StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[ - FunctionCallContent( - id=event.item_id, - name=event.call_id, - arguments=event.delta, - index=event.output_index, - metadata={"call_id": event.call_id}, + case FunctionResultContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function result needs to have a call_id") + continue + await self.connection.conversation.item.create( + item=ConversationItemParam( + type="function_call_output", + output=item.result, + call_id=call_id, ) - ], - choice_index=0, - inner_content=event, - ) - function_calls.append(msg) - yield [msg] - continue - case "response.function_call_arguments.done": - # execute function, add result to conversation - if len(function_calls) > 0: - function_call = sum(function_calls[1:], function_calls[0]) - # execute function - results = [] - for item in function_call.items: - if isinstance(item, FunctionCallContent): - kernel: Kernel | None = kwargs.get("kernel") - call_id = item.name - function_name = next( - output_item_event.item.name - for output_item_event in detailed_events["response.output_item.added"] - if output_item_event.item.call_id == call_id - ) - item.plugin_name, item.function_name = function_name.split("-", 1) - if kernel: - await kernel.invoke_function_call(item, chat_history) - # add result to conversation - results.append(chat_history.messages[-1]) - for message in results: - await self._add_content_to_conversation(content=message) - case _: - logger.debug("Unhandled event type: %s", event.type) - logger.debug(f"Finished streaming chat message contents, {len(events)} events received.") - for event_type in detailed_events: - logger.debug(f"Event type: {event_type}, count: {len(detailed_events[event_type])}") - - async def send_content( + ) + case SendEvents.CONVERSATION_ITEM_TRUNCATE: + if "item_id" not in kwargs: + logger.error("Event data does not contain 'item_id'") + return + await self.connection.conversation.item.truncate( + item_id=kwargs["item_id"], content_index=0, audio_end_ms=kwargs.get("audio_end_ms", 0) + ) + case SendEvents.CONVERSATION_ITEM_DELETE: + if "item_id" not in kwargs: + logger.error("Event data does not contain 'item_id'") + return + await self.connection.conversation.item.delete(item_id=kwargs["item_id"]) + case SendEvents.RESPONSE_CREATE: + if "response" in kwargs: + await self.connection.response.create(response=kwargs["response"]) + else: + await self.connection.response.create() + case SendEvents.RESPONSE_CANCEL: + if "response_id" in kwargs: + await self.connection.response.cancel(response_id=kwargs["response_id"]) + else: + await self.connection.response.cancel() + + @override + async def create_session( self, - content: ChatMessageContent | AudioContent | AsyncGenerator[AudioContent, Any], + settings: PromptExecutionSettings | None = None, + chat_history: ChatHistory | None = None, **kwargs: Any, ) -> None: - """Send a chat message content to the service. - - This content should contain audio content, either as a ChatMessageContent with a - AudioContent item, as AudioContent directly, as or as a generator of AudioContent. - - """ - if isinstance(content, AudioContent | ChatMessageContent): - if isinstance(content, ChatMessageContent): - content = next(item for item in content.items if isinstance(item, AudioContent)) - connection = await self._get_connection() - await connection.input_audio_buffer.append(audio=content.data.decode("utf-8")) - await asyncio.sleep(0) - return - - async for audio_content in content: - if isinstance(audio_content, ChatMessageContent): - audio_content = next(item for item in audio_content.items if isinstance(item, AudioContent)) - connection = await self._get_connection() - await connection.input_audio_buffer.append(audio=audio_content.data.decode("utf-8")) - await asyncio.sleep(0) - - async def commit_content(self, settings: "PromptExecutionSettings") -> None: - """Commit the chat message content to the service. - - This is only needed when turn detection is not handled by the service. - - This behavior is determined by the turn_detection parameter in the settings. - If turn_detection is None, then it will commit the audio buffer and - ask the service to process the audio and create the response. - """ - if not isinstance(settings, self.get_prompt_execution_settings_class()): - settings = self.get_prompt_execution_settings_from_settings(settings) - if not settings.turn_detection: - connection = await self._get_connection() - await connection.input_audio_buffer.commit() - await connection.response.create() + """Create a session in the service.""" + self.connection = await self.client.beta.realtime.connect(model=self.ai_model_id).enter() + self.connected.set() + if settings or chat_history or kwargs: + await self.update_session(settings=settings, chat_history=chat_history, **kwargs) @override - def _update_function_choice_settings_callback( + async def update_session( + self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any + ) -> None: + if settings: + if "kernel" in kwargs: + settings = prepare_settings_for_function_calling( + settings, + self.get_prompt_execution_settings_class(), + self._update_function_choice_settings_callback(), + kernel=kwargs.get("kernel"), # type: ignore + ) + await self.send_event(SendEvents.SESSION_UPDATE, settings=settings) + if chat_history and len(chat_history) > 0: + await asyncio.gather( + *(self.send_event(SendEvents.CONVERSATION_ITEM_CREATE, item=msg) for msg in chat_history.messages) + ) + + @override + async def close_session(self) -> None: + """Close the session in the service.""" + if self.connected.is_set(): + await self.connection.close() + self.connection = None + self.connected.clear() + + def response_audio_delta_callback( self, - ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: - return update_settings_from_function_call_configuration + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response audio delta.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[AudioContent(data=base64.b64decode(event.delta), data_format="base64")], + choice_index=event.content_index, + inner_content=event, + ), True - async def _streaming_function_call_result_callback( - self, function_result_messages: list[StreamingChatMessageContent] + def response_audio_transcript_delta_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response audio transcript delta.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[StreamingTextContent(text=event.delta, choice_index=event.content_index)], + choice_index=event.content_index, + inner_content=event, + ), True + + def response_audio_transcript_done_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response audio transcript done.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[StreamingTextContent(text=event.transcript, choice_index=event.content_index)], + choice_index=event.content_index, + inner_content=event, + ), False + + def response_function_call_arguments_delta_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response function call arguments delta.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + ], + choice_index=0, + inner_content=event, + ), True + + def error_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, ) -> None: - """Callback to handle the streaming function call result messages. - - Override this method to handle the streaming function call result messages. - - Args: - function_result_messages (list): The streaming function call result messages. - """ - for msg in function_result_messages: - await self._add_content_to_conversation(msg) - - async def _add_content_to_conversation(self, content: ChatMessageContent) -> None: - """Add an item to the conversation.""" - connection = await self._get_connection() - for item in content.items: - match item: - case AudioContent(): - await connection.conversation.item.create( - item=ConversationItemParam( - type="message", - content=[ - { - "type": "input_audio", - "audio": item.data.decode("utf-8"), - } - ], - role="user", - ) - ) - case TextContent(): - await connection.conversation.item.create( - item=ConversationItemParam( - type="message", - content=[ - { - "type": "input_text", - "text": item.text, - } - ], - role="user", - ) - ) - case FunctionCallContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function call needs to have a call_id") - continue - await connection.conversation.item.create( - item=ConversationItemParam( - type="function_call", - name=item.name, - arguments=item.arguments, - call_id=call_id, - ) - ) - case FunctionResultContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function result needs to have a call_id") - continue - await connection.conversation.item.create( - item=ConversationItemParam( - type="function_call_output", - output=item.result, - call_id=call_id, - ) - ) - case _: - logger.debug("Unhandled item type: %s", item.__class__.__name__) - continue + """Handle error.""" + logger.error("Error received: %s", event.error) + + def session_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> None: + """Handle session.""" + logger.debug("Session created or updated, session: %s", event.session) + + async def response_function_call_arguments_done_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> None: + """Handle response function call done.""" + item = FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + kernel: Kernel | None = kwargs.get("kernel") + call_id = item.name + function_name = next( + output_item_event.item.name + for output_item_event in self.event_log[ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED] + if output_item_event.item.call_id == call_id + ) + item.plugin_name, item.function_name = function_name.split("-", 1) + if kernel: + chat_history = ChatHistory() + await kernel.invoke_function_call(item, chat_history) + await self.send_event(SendEvents.CONVERSATION_ITEM_CREATE, item=chat_history.messages[-1]) + return chat_history.messages[-1], False + + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: + """Get the request settings class.""" + from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa + OpenAIRealtimeExecutionSettings, + ) + + return OpenAIRealtimeExecutionSettings diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index 734e7e7caed4..c5d092d50870 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -1,51 +1,140 @@ # Copyright (c) Microsoft. All rights reserved. - from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator -from typing import Any +from collections.abc import AsyncGenerator, Callable +from typing import TYPE_CHECKING, Any, ClassVar -from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings -from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType from semantic_kernel.services.ai_service_client_base import AIServiceClientBase +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings + from semantic_kernel.contents.chat_history import ChatHistory + from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent + +#### +# TODO (eavanvalkenburg): Move to ADR +# Receiving: +# Option 1: Events and Contents split (current) +# - content received through main receive_content method +# - events received through event callback handlers +# Option 2: Everything is Content +# - content (events as new Content Type) received through main receive_content method +# Option 3: Everything is Event +# - receive_content method is removed +# - events received through main listen method +# - default event handlers added for things like errors and function calling +# - built-in vs custom event handling - separate or not? +# Sending: +# Option 1: Events and Contents split (current) +# - send_content and send_event +# Option 2: Everything is Content +# - single method needed, with EventContent type support +# Option 3: Everything is Event +# - send_event method only, Content is part of event data +#### +@experimental_class class RealtimeClientBase(AIServiceClientBase, ABC): - """Base class for audio to text client.""" + """Base class for a realtime client.""" + + SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False + + async def __aenter__(self) -> "RealtimeClientBase": + """Enter the context manager. + + Default implementation calls the create session method. + """ + await self.create_session() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + """Exit the context manager.""" + await self.close_session() + + @abstractmethod + async def close_session(self) -> None: + """Close the session in the service.""" + pass @abstractmethod - async def receive( + async def create_session( self, - settings: PromptExecutionSettings | None = None, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, **kwargs: Any, - ) -> AsyncGenerator[TextContent | AudioContent, Any]: - """Get text contents from audio. + ) -> None: + """Create a session in the service. Args: settings: Prompt execution settings. + chat_history: Chat history. kwargs: Additional arguments. - - Returns: - list[TextContent | AudioContent]: response contents. """ raise NotImplementedError @abstractmethod - async def send( + async def update_session( self, - audio_content: AudioContent, - settings: PromptExecutionSettings | None = None, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> None: - """Get text content from audio. + """Update a session in the service. + + Can be used when using the context manager instead of calling create_session with these same arguments. Args: - audio_content: Audio content. settings: Prompt execution settings. + chat_history: Chat history. kwargs: Additional arguments. + """ + raise NotImplementedError - Returns: - TextContent: Text content. + @abstractmethod + async def event_listener( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + **kwargs: Any, + ) -> AsyncGenerator["StreamingChatMessageContent", Any]: + """Get text contents from audio. + + Args: + settings: Prompt execution settings. + chat_history: Chat history. + kwargs: Additional arguments. + + Yields: + StreamingChatMessageContent messages """ raise NotImplementedError + + @abstractmethod + async def send_event( + self, + event: str, + event_data: dict[str, Any] | None = None, + **kwargs: Any, + ) -> None: + """Send an event to the session. + + Args: + event: Event name, can be a string or a Enum value. + event_data: Event data. + kwargs: Additional arguments. + """ + raise NotImplementedError + + def _update_function_choice_settings_callback( + self, + ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: + """Return the callback function to update the settings from a function call configuration. + + Override this method to provide a custom callback function to + update the settings from a function call configuration. + """ + return lambda configuration, settings, choice_type: None diff --git a/python/tests/unit/contents/test_audio_content.py b/python/tests/unit/contents/test_audio_content.py deleted file mode 100644 index 2af5a99b9e29..000000000000 --- a/python/tests/unit/contents/test_audio_content.py +++ /dev/null @@ -1,60 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -import os - -import pytest - -from semantic_kernel.contents.audio_content import AudioContent - -test_cases = [ - pytest.param(AudioContent(uri="http://test_uri"), id="uri"), - pytest.param(AudioContent(data=b"test_data", mime_type="image/jpeg", data_format="base64"), id="data"), - pytest.param(AudioContent(uri="http://test_uri", data=b"test_data", mime_type="image/jpeg"), id="both"), - pytest.param( - AudioContent.from_image_path( - image_path=os.path.join(os.path.dirname(__file__), "../../", "assets/sample_image.jpg") - ), - id="image_file", - ), -] - - -def test_create_uri(): - image = AudioContent(uri="http://test_uri") - assert str(image.uri) == "http://test_uri/" - - -def test_create_file_from_path(): - image_path = os.path.join(os.path.dirname(__file__), "../../", "assets/sample_image.jpg") - image = AudioContent.from_image_path(image_path=image_path) - assert image.mime_type == "image/jpeg" - assert image.data_uri.startswith("data:image/jpeg;") - assert image.data is not None - - -def test_create_data(): - image = AudioContent(data=b"test_data", mime_type="image/jpeg") - assert image.mime_type == "image/jpeg" - assert image.data == b"test_data" - - -def test_to_str_uri(): - image = AudioContent(uri="http://test_uri") - assert str(image) == "http://test_uri/" - - -def test_to_str_data(): - image = AudioContent(data=b"test_data", mime_type="image/jpeg", data_format="base64") - assert str(image) == "data:image/jpeg;base64,dGVzdF9kYXRh" - - -@pytest.mark.parametrize("image", test_cases) -def test_element_roundtrip(image): - element = image.to_element() - new_image = AudioContent.from_element(element) - assert new_image == image - - -@pytest.mark.parametrize("image", test_cases) -def test_to_dict(image): - assert image.to_dict() == {"type": "image_url", "image_url": {"url": str(image)}} diff --git a/python/uv.lock b/python/uv.lock index 301c71febcfb..b53948ef0d64 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -414,30 +414,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.35.92" +version = "1.35.95" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "s3transfer", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3f/de/a96f2aa9a5770932e5bc3a9d3a6b4e0270487d5846a3387d5f5148e4c974/boto3-1.35.92.tar.gz", hash = "sha256:f7851cb320dcb2a53fc73b4075187ec9b05d51291539601fa238623fdc0e8cd3", size = 111016 } +sdist = { url = "https://files.pythonhosted.org/packages/97/b5/b961eb4d803ade4c90113b254630482f59a5d89b84e6939c9d4c7893d0c7/boto3-1.35.95.tar.gz", hash = "sha256:d5671226819f6a78e31b1f37bd60f194afb8203254a543d06bdfb76de7d79236", size = 111014 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/9d/0f7ecfea26ba0524617f7cfbd0b188d963bbc3b4cf2d9c3441dffe310c30/boto3-1.35.92-py3-none-any.whl", hash = "sha256:786930d5f1cd13d03db59ff2abbb2b7ffc173fd66646d5d8bee07f316a5f16ca", size = 139179 }, + { url = "https://files.pythonhosted.org/packages/3a/e1/1910792d5eceff426bd9048c454766df720cb0fd26473907fbfd1c64d518/boto3-1.35.95-py3-none-any.whl", hash = "sha256:c81223488607457dacb7829ee0c99803c664593b34a2b0f86c71d421e7c3469a", size = 139182 }, ] [[package]] name = "botocore" -version = "1.35.92" +version = "1.35.95" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "python-dateutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/bf/e1/4f3d4e43d10a4070aa43c6d9c0cfd40fe53dbd1c81a31f237c29a86735a3/botocore-1.35.92.tar.gz", hash = "sha256:caa7d5d857fed5b3d694b89c45f82b9f938f840e90a4eb7bf50aa65da2ba8f82", size = 13494438 } +sdist = { url = "https://files.pythonhosted.org/packages/d8/b7/1cf5da213ce2e00a5bcd480a9355aa23f787e11ef63eecb637bd7e48deef/botocore-1.35.95.tar.gz", hash = "sha256:b03d2d7cc58a16aa96a7e8f21941b766e98abc6ea74f61a63de7dc26c03641d3", size = 13489115 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/6f/015482b4bb28e9edcde97b67ec2d40f84956e1b8c7b22254f58a461d357d/botocore-1.35.92-py3-none-any.whl", hash = "sha256:f94ae1e056a675bd67c8af98a6858d06e3927d974d6c712ed6e27bb1d11bee1d", size = 13300322 }, + { url = "https://files.pythonhosted.org/packages/cf/97/e001bbab0773b66a5512022cc26deb82b8743f16ba5662fe762019c4c52c/botocore-1.35.95-py3-none-any.whl", hash = "sha256:a672406f748ad6a5b2fb7ea0d8394539eb4fda5332fc5373467d232c4bb27b12", size = 13289333 }, ] [[package]] @@ -646,7 +646,7 @@ wheels = [ [[package]] name = "chromadb" -version = "0.6.3" +version = "0.5.20" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bcrypt", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -678,9 +678,9 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "uvicorn", extra = ["standard"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/39/cd/f0f2de3f466ff514fb6b58271c14f6d22198402bb5b71b8d890231265946/chromadb-0.6.3.tar.gz", hash = "sha256:c8f34c0b704b9108b04491480a36d42e894a960429f87c6516027b5481d59ed3", size = 29297929 } +sdist = { url = "https://files.pythonhosted.org/packages/03/31/6c8e05405bb02b4a1f71f9aa3eef242415565dabf6afc1bde7f64f726963/chromadb-0.5.20.tar.gz", hash = "sha256:19513a23b2d20059866216bfd80195d1d4a160ffba234b8899f5e80978160ca7", size = 33664540 } wheels = [ - { url = "https://files.pythonhosted.org/packages/28/8e/5c186c77bf749b6fe0528385e507e463f1667543328d76fd00a49e1a4e6a/chromadb-0.6.3-py3-none-any.whl", hash = "sha256:4851258489a3612b558488d98d09ae0fe0a28d5cad6bd1ba64b96fdc419dc0e5", size = 611129 }, + { url = "https://files.pythonhosted.org/packages/5f/7a/10bf5dc92d13cc03230190fcc5016a0b138d99e5b36b8b89ee0fe1680e10/chromadb-0.5.20-py3-none-any.whl", hash = "sha256:9550ba1b6dce911e35cac2568b301badf4b42f457b99a432bdeec2b6b9dd3680", size = 617884 }, ] [[package]] @@ -984,6 +984,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/a3/ac312faeceffd2d8f86bc6dcb5c401188ba5a01bc88e69bed97578a0dfcd/durationpy-0.9-py3-none-any.whl", hash = "sha256:e65359a7af5cedad07fb77a2dd3f390f8eb0b74cb845589fa6c057086834dd38", size = 3461 }, ] +[[package]] +name = "environs" +version = "9.5.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "marshmallow", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "python-dotenv", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/d4/e3/c3c6c76f3dbe3e019e9a451b35bf9f44690026a5bb1232f7b77097b72ff5/environs-9.5.0.tar.gz", hash = "sha256:a76307b36fbe856bdca7ee9161e6c466fd7fcffc297109a118c59b54e27e30c9", size = 20795 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/5e/f0f217dc393372681bfe05c50f06a212e78d0a3fee907a74ab451ec1dcdb/environs-9.5.0-py2.py3-none-any.whl", hash = "sha256:1e549569a3de49c05f856f40bce86979e7d5ffbbc4398e7f338574c220189124", size = 12548 }, +] + [[package]] name = "eval-type-backport" version = "0.2.2" @@ -1250,7 +1263,7 @@ wheels = [ [[package]] name = "google-cloud-aiplatform" -version = "1.75.0" +version = "1.76.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "docstring-parser", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -1266,9 +1279,9 @@ dependencies = [ { name = "shapely", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/de/76/7b3c013e92c70a558e71b0e83be13111ec797c4ded8ca98df20af15891c7/google_cloud_aiplatform-1.75.0.tar.gz", hash = "sha256:eb8404abf1134b3b368535fe429c4eec2fd12d444c2e9ffbc329ddcbc72b36c9", size = 8185280 } +sdist = { url = "https://files.pythonhosted.org/packages/49/61/c3f206a0de113cdba09998b78434c63fcabd8e89607b8fb83cd21a3dffcf/google_cloud_aiplatform-1.76.0.tar.gz", hash = "sha256:910fb7fb6ef7ec73a48523872d669370755f59ac6d764dc8bf2fc91e7c0b2fca", size = 8202679 } wheels = [ - { url = "https://files.pythonhosted.org/packages/06/d4/4b9df013c442e3b8db425924e896b5eaaeb23d1a036aa01002a3f83b936c/google_cloud_aiplatform-1.75.0-py2.py3-none-any.whl", hash = "sha256:eb5d79b5f7210d79a22b53c93a69b5bae5680dfc829387ea020765b97786b3d0", size = 6854342 }, + { url = "https://files.pythonhosted.org/packages/46/01/651752ae55160f5670c33d8a61de08798212472d11db124cb175ff54bcaa/google_cloud_aiplatform-1.76.0-py2.py3-none-any.whl", hash = "sha256:0b0348525b9528db7b69538ff6e86289ea2ce0d80f3784a42865fc994fe10dd1", size = 6867667 }, ] [[package]] @@ -1424,122 +1437,122 @@ wheels = [ [[package]] name = "grpcio" -version = "1.67.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/20/53/d9282a66a5db45981499190b77790570617a604a38f3d103d0400974aeb5/grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732", size = 12580022 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/cd/f6ca5c49aa0ae7bc6d0757f7dae6f789569e9490a635eaabe02bc02de7dc/grpcio-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:8b0341d66a57f8a3119b77ab32207072be60c9bf79760fa609c5609f2deb1f3f", size = 5112450 }, - { url = "https://files.pythonhosted.org/packages/d4/f0/d9bbb4a83cbee22f738ee7a74aa41e09ccfb2dcea2cc30ebe8dab5b21771/grpcio-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5a27dddefe0e2357d3e617b9079b4bfdc91341a91565111a21ed6ebbc51b22d", size = 10937518 }, - { url = "https://files.pythonhosted.org/packages/5b/17/0c5dbae3af548eb76669887642b5f24b232b021afe77eb42e22bc8951d9c/grpcio-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:43112046864317498a33bdc4797ae6a268c36345a910de9b9c17159d8346602f", size = 5633610 }, - { url = "https://files.pythonhosted.org/packages/17/48/e000614e00153d7b2760dcd9526b95d72f5cfe473b988e78f0ff3b472f6c/grpcio-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9b929f13677b10f63124c1a410994a401cdd85214ad83ab67cc077fc7e480f0", size = 6240678 }, - { url = "https://files.pythonhosted.org/packages/64/19/a16762a70eeb8ddfe43283ce434d1499c1c409ceec0c646f783883084478/grpcio-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7d1797a8a3845437d327145959a2c0c47c05947c9eef5ff1a4c80e499dcc6fa", size = 5884528 }, - { url = "https://files.pythonhosted.org/packages/6b/dc/bd016aa3684914acd2c0c7fa4953b2a11583c2b844f3d7bae91fa9b98fbb/grpcio-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0489063974d1452436139501bf6b180f63d4977223ee87488fe36858c5725292", size = 6583680 }, - { url = "https://files.pythonhosted.org/packages/1a/93/1441cb14c874f11aa798a816d582f9da82194b6677f0f134ea53d2d5dbeb/grpcio-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9fd042de4a82e3e7aca44008ee2fb5da01b3e5adb316348c21980f7f58adc311", size = 6162967 }, - { url = "https://files.pythonhosted.org/packages/29/e9/9295090380fb4339b7e935b9d005fa9936dd573a22d147c9e5bb2df1b8d4/grpcio-1.67.1-cp310-cp310-win32.whl", hash = "sha256:638354e698fd0c6c76b04540a850bf1db27b4d2515a19fcd5cf645c48d3eb1ed", size = 3616336 }, - { url = "https://files.pythonhosted.org/packages/ce/de/7c783b8cb8f02c667ca075c49680c4aeb8b054bc69784bcb3e7c1bbf4985/grpcio-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:608d87d1bdabf9e2868b12338cd38a79969eaf920c89d698ead08f48de9c0f9e", size = 4352071 }, - { url = "https://files.pythonhosted.org/packages/59/2c/b60d6ea1f63a20a8d09c6db95c4f9a16497913fb3048ce0990ed81aeeca0/grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb", size = 5119075 }, - { url = "https://files.pythonhosted.org/packages/b3/9a/e1956f7ca582a22dd1f17b9e26fcb8229051b0ce6d33b47227824772feec/grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e", size = 11009159 }, - { url = "https://files.pythonhosted.org/packages/43/a8/35fbbba580c4adb1d40d12e244cf9f7c74a379073c0a0ca9d1b5338675a1/grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f", size = 5629476 }, - { url = "https://files.pythonhosted.org/packages/77/c9/864d336e167263d14dfccb4dbfa7fce634d45775609895287189a03f1fc3/grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc", size = 6239901 }, - { url = "https://files.pythonhosted.org/packages/f7/1e/0011408ebabf9bd69f4f87cc1515cbfe2094e5a32316f8714a75fd8ddfcb/grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96", size = 5881010 }, - { url = "https://files.pythonhosted.org/packages/b4/7d/fbca85ee9123fb296d4eff8df566f458d738186d0067dec6f0aa2fd79d71/grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f", size = 6580706 }, - { url = "https://files.pythonhosted.org/packages/75/7a/766149dcfa2dfa81835bf7df623944c1f636a15fcb9b6138ebe29baf0bc6/grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970", size = 6161799 }, - { url = "https://files.pythonhosted.org/packages/09/13/5b75ae88810aaea19e846f5380611837de411181df51fd7a7d10cb178dcb/grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744", size = 3616330 }, - { url = "https://files.pythonhosted.org/packages/aa/39/38117259613f68f072778c9638a61579c0cfa5678c2558706b10dd1d11d3/grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5", size = 4354535 }, - { url = "https://files.pythonhosted.org/packages/6e/25/6f95bd18d5f506364379eabc0d5874873cc7dbdaf0757df8d1e82bc07a88/grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953", size = 5089809 }, - { url = "https://files.pythonhosted.org/packages/10/3f/d79e32e5d0354be33a12db2267c66d3cfeff700dd5ccdd09fd44a3ff4fb6/grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb", size = 10981985 }, - { url = "https://files.pythonhosted.org/packages/21/f2/36fbc14b3542e3a1c20fb98bd60c4732c55a44e374a4eb68f91f28f14aab/grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0", size = 5588770 }, - { url = "https://files.pythonhosted.org/packages/0d/af/bbc1305df60c4e65de8c12820a942b5e37f9cf684ef5e49a63fbb1476a73/grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af", size = 6214476 }, - { url = "https://files.pythonhosted.org/packages/92/cf/1d4c3e93efa93223e06a5c83ac27e32935f998bc368e276ef858b8883154/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e", size = 5850129 }, - { url = "https://files.pythonhosted.org/packages/ae/ca/26195b66cb253ac4d5ef59846e354d335c9581dba891624011da0e95d67b/grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75", size = 6568489 }, - { url = "https://files.pythonhosted.org/packages/d1/94/16550ad6b3f13b96f0856ee5dfc2554efac28539ee84a51d7b14526da985/grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38", size = 6149369 }, - { url = "https://files.pythonhosted.org/packages/33/0d/4c3b2587e8ad7f121b597329e6c2620374fccbc2e4e1aa3c73ccc670fde4/grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78", size = 3599176 }, - { url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574 }, - { url = "https://files.pythonhosted.org/packages/12/d2/2f032b7a153c7723ea3dea08bffa4bcaca9e0e5bdf643ce565b76da87461/grpcio-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa0162e56fd10a5547fac8774c4899fc3e18c1aa4a4759d0ce2cd00d3696ea6b", size = 5091487 }, - { url = "https://files.pythonhosted.org/packages/d0/ae/ea2ff6bd2475a082eb97db1104a903cf5fc57c88c87c10b3c3f41a184fc0/grpcio-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:beee96c8c0b1a75d556fe57b92b58b4347c77a65781ee2ac749d550f2a365dc1", size = 10943530 }, - { url = "https://files.pythonhosted.org/packages/07/62/646be83d1a78edf8d69b56647327c9afc223e3140a744c59b25fbb279c3b/grpcio-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:a93deda571a1bf94ec1f6fcda2872dad3ae538700d94dc283c672a3b508ba3af", size = 5589079 }, - { url = "https://files.pythonhosted.org/packages/d0/25/71513d0a1b2072ce80d7f5909a93596b7ed10348b2ea4fdcbad23f6017bf/grpcio-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e6f255980afef598a9e64a24efce87b625e3e3c80a45162d111a461a9f92955", size = 6213542 }, - { url = "https://files.pythonhosted.org/packages/76/9a/d21236297111052dcb5dc85cd77dc7bf25ba67a0f55ae028b2af19a704bc/grpcio-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e838cad2176ebd5d4a8bb03955138d6589ce9e2ce5d51c3ada34396dbd2dba8", size = 5850211 }, - { url = "https://files.pythonhosted.org/packages/2d/fe/70b1da9037f5055be14f359026c238821b9bcf6ca38a8d760f59a589aacd/grpcio-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a6703916c43b1d468d0756c8077b12017a9fcb6a1ef13faf49e67d20d7ebda62", size = 6572129 }, - { url = "https://files.pythonhosted.org/packages/74/0d/7df509a2cd2a54814598caf2fb759f3e0b93764431ff410f2175a6efb9e4/grpcio-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:917e8d8994eed1d86b907ba2a61b9f0aef27a2155bca6cbb322430fc7135b7bb", size = 6149819 }, - { url = "https://files.pythonhosted.org/packages/0a/08/bc3b0155600898fd10f16b79054e1cca6cb644fa3c250c0fe59385df5e6f/grpcio-1.67.1-cp313-cp313-win32.whl", hash = "sha256:e279330bef1744040db8fc432becc8a727b84f456ab62b744d3fdb83f327e121", size = 3596561 }, - { url = "https://files.pythonhosted.org/packages/5a/96/44759eca966720d0f3e1b105c43f8ad4590c97bf8eb3cd489656e9590baa/grpcio-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:fa0c739ad8b1996bd24823950e3cb5152ae91fca1c09cc791190bf1627ffefba", size = 4346042 }, +version = "1.69.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e4/87/06a145284cbe86c91ca517fe6b57be5efbb733c0d6374b407f0992054d18/grpcio-1.69.0.tar.gz", hash = "sha256:936fa44241b5379c5afc344e1260d467bee495747eaf478de825bab2791da6f5", size = 12738244 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b0/6e/2f8ee5fb65aef962d0bd7e46b815e7b52820687e29c138eaee207a688abc/grpcio-1.69.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2060ca95a8db295ae828d0fc1c7f38fb26ccd5edf9aa51a0f44251f5da332e97", size = 5190753 }, + { url = "https://files.pythonhosted.org/packages/89/07/028dcda44d40f9488f0a0de79c5ffc80e2c1bc5ed89da9483932e3ea67cf/grpcio-1.69.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2e52e107261fd8fa8fa457fe44bfadb904ae869d87c1280bf60f93ecd3e79278", size = 11096752 }, + { url = "https://files.pythonhosted.org/packages/99/a0/c727041b1410605ba38b585b6b52c1a289d7fcd70a41bccbc2c58fc643b2/grpcio-1.69.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:316463c0832d5fcdb5e35ff2826d9aa3f26758d29cdfb59a368c1d6c39615a11", size = 5705442 }, + { url = "https://files.pythonhosted.org/packages/7a/2f/1c53f5d127ff882443b19c757d087da1908f41c58c4b098e8eaf6b2bb70a/grpcio-1.69.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26c9a9c4ac917efab4704b18eed9082ed3b6ad19595f047e8173b5182fec0d5e", size = 6333796 }, + { url = "https://files.pythonhosted.org/packages/cc/f6/2017da2a1b64e896af710253e5bfbb4188605cdc18bce3930dae5cdbf502/grpcio-1.69.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90b3646ced2eae3a0599658eeccc5ba7f303bf51b82514c50715bdd2b109e5ec", size = 5954245 }, + { url = "https://files.pythonhosted.org/packages/c1/65/1395bec928e99ba600464fb01b541e7e4cdd462e6db25259d755ef9f8d02/grpcio-1.69.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3b75aea7c6cb91b341c85e7c1d9db1e09e1dd630b0717f836be94971e015031e", size = 6664854 }, + { url = "https://files.pythonhosted.org/packages/40/57/8b3389cfeb92056c8b44288c9c4ed1d331bcad0215c4eea9ae4629e156d9/grpcio-1.69.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5cfd14175f9db33d4b74d63de87c64bb0ee29ce475ce3c00c01ad2a3dc2a9e51", size = 6226854 }, + { url = "https://files.pythonhosted.org/packages/cc/61/1f2bbeb7c15544dffc98b3f65c093e746019995e6f1e21dc3655eec3dc23/grpcio-1.69.0-cp310-cp310-win32.whl", hash = "sha256:9031069d36cb949205293cf0e243abd5e64d6c93e01b078c37921493a41b72dc", size = 3662734 }, + { url = "https://files.pythonhosted.org/packages/ef/ba/bf1a6d9f5c17d2da849793d72039776c56c98c889c9527f6721b6ee57e6e/grpcio-1.69.0-cp310-cp310-win_amd64.whl", hash = "sha256:cc89b6c29f3dccbe12d7a3b3f1b3999db4882ae076c1c1f6df231d55dbd767a5", size = 4410306 }, + { url = "https://files.pythonhosted.org/packages/8d/cd/ca256aeef64047881586331347cd5a68a4574ba1a236e293cd8eba34e355/grpcio-1.69.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8de1b192c29b8ce45ee26a700044717bcbbd21c697fa1124d440548964328561", size = 5198734 }, + { url = "https://files.pythonhosted.org/packages/37/3f/10c1e5e0150bf59aa08ea6aebf38f87622f95f7f33f98954b43d1b2a3200/grpcio-1.69.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:7e76accf38808f5c5c752b0ab3fd919eb14ff8fafb8db520ad1cc12afff74de6", size = 11135285 }, + { url = "https://files.pythonhosted.org/packages/08/61/61cd116a572203a740684fcba3fef37a3524f1cf032b6568e1e639e59db0/grpcio-1.69.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:d5658c3c2660417d82db51e168b277e0ff036d0b0f859fa7576c0ffd2aec1442", size = 5699468 }, + { url = "https://files.pythonhosted.org/packages/01/f1/a841662e8e2465ba171c973b77d18fa7438ced535519b3c53617b7e6e25c/grpcio-1.69.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5494d0e52bf77a2f7eb17c6da662886ca0a731e56c1c85b93505bece8dc6cf4c", size = 6332337 }, + { url = "https://files.pythonhosted.org/packages/62/b1/c30e932e02c2e0bfdb8df46fe3b0c47f518fb04158ebdc0eb96cc97d642f/grpcio-1.69.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed866f9edb574fd9be71bf64c954ce1b88fc93b2a4cbf94af221e9426eb14d6", size = 5949844 }, + { url = "https://files.pythonhosted.org/packages/5e/cb/55327d43b6286100ffae7d1791be6178d13c917382f3e9f43f82e8b393cf/grpcio-1.69.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c5ba38aeac7a2fe353615c6b4213d1fbb3a3c34f86b4aaa8be08baaaee8cc56d", size = 6661828 }, + { url = "https://files.pythonhosted.org/packages/6f/e4/120d72ae982d51cb9cabcd9672f8a1c6d62011b493a4d049d2abdf564db0/grpcio-1.69.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f79e05f5bbf551c4057c227d1b041ace0e78462ac8128e2ad39ec58a382536d2", size = 6226026 }, + { url = "https://files.pythonhosted.org/packages/96/e8/2cc15f11db506d7b1778f0587fa7bdd781602b05b3c4d75b7ca13de33d62/grpcio-1.69.0-cp311-cp311-win32.whl", hash = "sha256:bf1f8be0da3fcdb2c1e9f374f3c2d043d606d69f425cd685110dd6d0d2d61258", size = 3662653 }, + { url = "https://files.pythonhosted.org/packages/42/78/3c5216829a48237fcb71a077f891328a435e980d9757a9ebc49114d88768/grpcio-1.69.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb9302afc3a0e4ba0b225cd651ef8e478bf0070cf11a529175caecd5ea2474e7", size = 4412824 }, + { url = "https://files.pythonhosted.org/packages/61/1d/8f28f147d7f3f5d6b6082f14e1e0f40d58e50bc2bd30d2377c730c57a286/grpcio-1.69.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:fc18a4de8c33491ad6f70022af5c460b39611e39578a4d84de0fe92f12d5d47b", size = 5161414 }, + { url = "https://files.pythonhosted.org/packages/35/4b/9ab8ea65e515e1844feced1ef9e7a5d8359c48d986c93f3d2a2006fbdb63/grpcio-1.69.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:0f0270bd9ffbff6961fe1da487bdcd594407ad390cc7960e738725d4807b18c4", size = 11108909 }, + { url = "https://files.pythonhosted.org/packages/99/68/1856fde2b3c3162bdfb9845978608deef3606e6907fdc2c87443fce6ecd0/grpcio-1.69.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:dc48f99cc05e0698e689b51a05933253c69a8c8559a47f605cff83801b03af0e", size = 5658302 }, + { url = "https://files.pythonhosted.org/packages/3e/21/3fa78d38dc5080d0d677103fad3a8cd55091635cc2069a7c06c7a54e6c4d/grpcio-1.69.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e925954b18d41aeb5ae250262116d0970893b38232689c4240024e4333ac084", size = 6306201 }, + { url = "https://files.pythonhosted.org/packages/f3/cb/5c47b82fd1baf43dba973ae399095d51aaf0085ab0439838b4cbb1e87e3c/grpcio-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87d222569273720366f68a99cb62e6194681eb763ee1d3b1005840678d4884f9", size = 5919649 }, + { url = "https://files.pythonhosted.org/packages/c6/67/59d1a56a0f9508a29ea03e1ce800bdfacc1f32b4f6b15274b2e057bf8758/grpcio-1.69.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b62b0f41e6e01a3e5082000b612064c87c93a49b05f7602fe1b7aa9fd5171a1d", size = 6648974 }, + { url = "https://files.pythonhosted.org/packages/f8/fe/ca70c14d98c6400095f19a0f4df8273d09c2106189751b564b26019f1dbe/grpcio-1.69.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db6f9fd2578dbe37db4b2994c94a1d9c93552ed77dca80e1657bb8a05b898b55", size = 6215144 }, + { url = "https://files.pythonhosted.org/packages/b3/94/b2b0a9fd487fc8262e20e6dd0ec90d9fa462c82a43b4855285620f6e9d01/grpcio-1.69.0-cp312-cp312-win32.whl", hash = "sha256:b192b81076073ed46f4b4dd612b8897d9a1e39d4eabd822e5da7b38497ed77e1", size = 3644552 }, + { url = "https://files.pythonhosted.org/packages/93/99/81aec9f85412e3255a591ae2ccb799238e074be774e5f741abae08a23418/grpcio-1.69.0-cp312-cp312-win_amd64.whl", hash = "sha256:1227ff7836f7b3a4ab04e5754f1d001fa52a730685d3dc894ed8bc262cc96c01", size = 4399532 }, + { url = "https://files.pythonhosted.org/packages/54/47/3ff4501365f56b7cc16617695dbd4fd838c5e362bc7fa9fee09d592f7d78/grpcio-1.69.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:a78a06911d4081a24a1761d16215a08e9b6d4d29cdbb7e427e6c7e17b06bcc5d", size = 5162928 }, + { url = "https://files.pythonhosted.org/packages/c0/63/437174c5fa951052c9ecc5f373f62af6f3baf25f3f5ef35cbf561806b371/grpcio-1.69.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:dc5a351927d605b2721cbb46158e431dd49ce66ffbacb03e709dc07a491dde35", size = 11103027 }, + { url = "https://files.pythonhosted.org/packages/53/df/53566a6fdc26b6d1f0585896e1cc4825961039bca5a6a314ff29d79b5d5b/grpcio-1.69.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:3629d8a8185f5139869a6a17865d03113a260e311e78fbe313f1a71603617589", size = 5659277 }, + { url = "https://files.pythonhosted.org/packages/e6/4c/b8a0c4f71498b6f9be5ca6d290d576cf2af9d95fd9827c47364f023969ad/grpcio-1.69.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9a281878feeb9ae26db0622a19add03922a028d4db684658f16d546601a4870", size = 6305255 }, + { url = "https://files.pythonhosted.org/packages/ef/55/d9aa05eb3dfcf6aa946aaf986740ec07fc5189f20e2cbeb8c5d278ffd00f/grpcio-1.69.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc614e895177ab7e4b70f154d1a7c97e152577ea101d76026d132b7aaba003b", size = 5920240 }, + { url = "https://files.pythonhosted.org/packages/ea/eb/774b27c51e3e386dfe6c491a710f6f87ffdb20d88ec6c3581e047d9354a2/grpcio-1.69.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:1ee76cd7e2e49cf9264f6812d8c9ac1b85dda0eaea063af07292400f9191750e", size = 6652974 }, + { url = "https://files.pythonhosted.org/packages/59/98/96de14e6e7d89123813d58c246d9b0f1fbd24f9277f5295264e60861d9d6/grpcio-1.69.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:0470fa911c503af59ec8bc4c82b371ee4303ececbbdc055f55ce48e38b20fd67", size = 6215757 }, + { url = "https://files.pythonhosted.org/packages/7d/5b/ce922e0785910b10756fabc51fd294260384a44bea41651dadc4e47ddc82/grpcio-1.69.0-cp313-cp313-win32.whl", hash = "sha256:b650f34aceac8b2d08a4c8d7dc3e8a593f4d9e26d86751ebf74ebf5107d927de", size = 3642488 }, + { url = "https://files.pythonhosted.org/packages/5d/04/11329e6ca1ceeb276df2d9c316b5e170835a687a4d0f778dba8294657e36/grpcio-1.69.0-cp313-cp313-win_amd64.whl", hash = "sha256:028337786f11fecb5d7b7fa660475a06aabf7e5e52b5ac2df47414878c0ce7ea", size = 4399968 }, ] [[package]] name = "grpcio-health-checking" -version = "1.67.1" +version = "1.69.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/dd/e3b339fa44dc75b501a1a22cb88f1af5b1f8c964488f19c4de4cfbbf05ba/grpcio_health_checking-1.67.1.tar.gz", hash = "sha256:ca90fa76a6afbb4fda71d734cb9767819bba14928b91e308cffbb0c311eb941e", size = 16775 } +sdist = { url = "https://files.pythonhosted.org/packages/ef/b8/d6d485e27d60174ba22c25587c1a97512c6a800633cfd6a8cd7943ad66e0/grpcio_health_checking-1.69.0.tar.gz", hash = "sha256:ff6e1d38c2a300b1bbd296916fbd9165667bc4b5a8557f99dd4226d4f9e8f4c1", size = 16809 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5c/8d/7a9878dca6616b48093d71c52d0bc79cb2dd1a2698ff6f5ce7406306de12/grpcio_health_checking-1.67.1-py3-none-any.whl", hash = "sha256:93753da5062152660aef2286c9b261e07dd87124a65e4dc9fbd47d1ce966b39d", size = 18924 }, + { url = "https://files.pythonhosted.org/packages/a4/07/8d68bb1821dc46dfb5b702374c5d06e9c0013afb08fa92516ebd8f963ef3/grpcio_health_checking-1.69.0-py3-none-any.whl", hash = "sha256:d2d0eec7e3af245863fd4997e2942d27c0868fbd61ffa4d14bc492c3e2c67127", size = 18923 }, ] [[package]] name = "grpcio-status" -version = "1.67.1" +version = "1.69.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/be/c7/fe0e79a80ac6346e0c6c0a24e9e3cbc3ae1c2a009acffb59eab484a6f69b/grpcio_status-1.67.1.tar.gz", hash = "sha256:2bf38395e028ceeecfd8866b081f61628114b384da7d51ae064ddc8d766a5d11", size = 13673 } +sdist = { url = "https://files.pythonhosted.org/packages/02/35/52dc0d8300f879dbf9cdc95764cee9f56d5a212998cfa1a8871b262df2a4/grpcio_status-1.69.0.tar.gz", hash = "sha256:595ef84e5178d6281caa732ccf68ff83259241608d26b0e9c40a5e66eee2a2d2", size = 13662 } wheels = [ - { url = "https://files.pythonhosted.org/packages/05/18/56999a1da3577d8ccc8698a575d6638e15fe25650cc88b2ce0a087f180b9/grpcio_status-1.67.1-py3-none-any.whl", hash = "sha256:16e6c085950bdacac97c779e6a502ea671232385e6e37f258884d6883392c2bd", size = 14427 }, + { url = "https://files.pythonhosted.org/packages/f6/e2/346a766a4232f74f45f8bc70e636fc3a6677e6bc3893382187829085f12e/grpcio_status-1.69.0-py3-none-any.whl", hash = "sha256:d6b2a3c9562c03a817c628d7ba9a925e209c228762d6d7677ae5c9401a542853", size = 14428 }, ] [[package]] name = "grpcio-tools" -version = "1.67.1" +version = "1.69.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "setuptools", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/6facde12a5a8da4398a3a8947f8ba6ef33b408dfc9767c8cefc0074ddd68/grpcio_tools-1.67.1.tar.gz", hash = "sha256:d9657f5ddc62b52f58904e6054b7d8a8909ed08a1e28b734be3a707087bcf004", size = 5159073 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e5/46/668e681e2e4ca7dc80cb5ad22bc794958c8b604b5b3143f16b94be3c0118/grpcio_tools-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:c701aaa51fde1f2644bd94941aa94c337adb86f25cd03cf05e37387aaea25800", size = 2308117 }, - { url = "https://files.pythonhosted.org/packages/d6/56/1c65fb7c836cd40470f1f1a88185973466241fdb42b42b7a83367c268622/grpcio_tools-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:6a722bba714392de2386569c40942566b83725fa5c5450b8910e3832a5379469", size = 5500152 }, - { url = "https://files.pythonhosted.org/packages/01/ab/caf9c330241d843a83043b023e2996e959cdc2c3ab404b1a9938eb734143/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0c7415235cb154e40b5ae90e2a172a0eb8c774b6876f53947cf0af05c983d549", size = 2282055 }, - { url = "https://files.pythonhosted.org/packages/75/e6/0cd849d140b58fedb7d3b15d907fe2eefd4dadff09b570dd687d841c5d00/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4c459098c4934f9470280baf9ff8b38c365e147f33c8abc26039a948a664a5", size = 2617360 }, - { url = "https://files.pythonhosted.org/packages/b9/51/bd73cd6515c2e81ba0a29b3cf6f2f62ad94737326f70b32511d1972a383e/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e89bf53a268f55c16989dab1cf0b32a5bff910762f138136ffad4146129b7a10", size = 2416028 }, - { url = "https://files.pythonhosted.org/packages/47/e5/6a16e23036f625b6d60b579996bb9bb7165485903f934d9d9d73b3f03ef5/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f09cb3e6bcb140f57b878580cf3b848976f67faaf53d850a7da9bfac12437068", size = 3224906 }, - { url = "https://files.pythonhosted.org/packages/14/cb/230c17d4372fa46fc799a822f25fa00c8eb3f85cc86e192b9606a17f732f/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:616dd0c6686212ca90ff899bb37eb774798677e43dc6f78c6954470782d37399", size = 2870384 }, - { url = "https://files.pythonhosted.org/packages/66/fd/6d9dd3bf5982ab7d7e773f055360185e96a96cf95f2cbc7f53ded5912ef5/grpcio_tools-1.67.1-cp310-cp310-win32.whl", hash = "sha256:58a66dbb3f0fef0396737ac09d6571a7f8d96a544ce3ed04c161f3d4fa8d51cc", size = 941138 }, - { url = "https://files.pythonhosted.org/packages/6a/97/2fd5ebd996c12b2cb1e1202ee4a03cac0a65ba17d29dd34253bfe2079839/grpcio_tools-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:89ee7c505bdf152e67c2cced6055aed4c2d4170f53a2b46a7e543d3b90e7b977", size = 1091151 }, - { url = "https://files.pythonhosted.org/packages/b5/9a/ec06547673c5001c2604637069ff8f287df1aef3f0f8809b09a1c936b049/grpcio_tools-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:6d80ddd87a2fb7131d242f7d720222ef4f0f86f53ec87b0a6198c343d8e4a86e", size = 2307990 }, - { url = "https://files.pythonhosted.org/packages/ca/84/4b7c3c27a2972c00b3b6ccaadd349e0f86b7039565d3a4932e219a4d76e0/grpcio_tools-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b655425b82df51f3bd9fd3ba1a6282d5c9ce1937709f059cb3d419b224532d89", size = 5526552 }, - { url = "https://files.pythonhosted.org/packages/a7/2d/a620e4c53a3b808ebecaa5033c2176925ee1c6cbb45c29af8bec9a249822/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:250241e6f9d20d0910a46887dfcbf2ec9108efd3b48f3fb95bb42d50d09d03f8", size = 2282137 }, - { url = "https://files.pythonhosted.org/packages/ec/29/e188b2e438781b37532abb8f10caf5b09c611a0bf9a09940b4cf303afd5b/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6008f5a5add0b6f03082edb597acf20d5a9e4e7c55ea1edac8296c19e6a0ec8d", size = 2617333 }, - { url = "https://files.pythonhosted.org/packages/86/aa/2bbccd3c34b1fa48b892fbad91525c33a8aa85cbedd50e8b0d17dc260dc3/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5eff9818c3831fa23735db1fa39aeff65e790044d0a312260a0c41ae29cc2d9e", size = 2415806 }, - { url = "https://files.pythonhosted.org/packages/db/34/99853a8ced1119937d02511476018dc1d6b295a4803d4ead5dbf9c55e9bc/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:262ab7c40113f8c3c246e28e369661ddf616a351cb34169b8ba470c9a9c3b56f", size = 3224765 }, - { url = "https://files.pythonhosted.org/packages/66/39/8537a8ace8f6242f2058677e56a429587ec731c332985af34f35d496ca58/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1eebd8c746adf5786fa4c3056258c21cc470e1eca51d3ed23a7fb6a697fe4e81", size = 2870446 }, - { url = "https://files.pythonhosted.org/packages/28/2a/5c04375adccff58647d48675e055895c31811a0ad896e4ba310833e2154d/grpcio_tools-1.67.1-cp311-cp311-win32.whl", hash = "sha256:3eff92fb8ca1dd55e3af0ef02236c648921fb7d0e8ca206b889585804b3659ae", size = 940890 }, - { url = "https://files.pythonhosted.org/packages/e6/ee/7861339c2cec8d55a5e859cf3682bda34eab5a040f95d0c80f775d6a3279/grpcio_tools-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:1ed18281ee17e5e0f9f6ce0c6eb3825ca9b5a0866fc1db2e17fab8aca28b8d9f", size = 1091094 }, - { url = "https://files.pythonhosted.org/packages/d9/cf/7b1908ca72e484bac555431036292c48d2d6504a45e2789848cb5ff313a8/grpcio_tools-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:bd5caef3a484e226d05a3f72b2d69af500dca972cf434bf6b08b150880166f0b", size = 2307645 }, - { url = "https://files.pythonhosted.org/packages/bb/15/0d1efb38af8af7e56b2342322634a3caf5f1337a6c3857a6d14aa590dfdf/grpcio_tools-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:48a2d63d1010e5b218e8e758ecb2a8d63c0c6016434e9f973df1c3558917020a", size = 5525468 }, - { url = "https://files.pythonhosted.org/packages/52/42/a810709099f09ade7f32990c0712c555b3d7eab6a05fb62618c17f8fe9da/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:baa64a6aa009bffe86309e236c81b02cd4a88c1ebd66f2d92e84e9b97a9ae857", size = 2281768 }, - { url = "https://files.pythonhosted.org/packages/4c/2a/64ee6cfdf1c32ef8bdd67bf04ae2f745f517f4a546281453ca1f68fa79ca/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ab318c40b5e3c097a159035fc3e4ecfbe9b3d2c9de189e55468b2c27639a6ab", size = 2617359 }, - { url = "https://files.pythonhosted.org/packages/79/7f/1ed8cd1529253fef9cf0ef3cd8382641125a5ca2eaa08eaffbb549f84e0b/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50eba3e31f9ac1149463ad9182a37349850904f142cffbd957cd7f54ec320b8e", size = 2415323 }, - { url = "https://files.pythonhosted.org/packages/8e/08/59f0073c58703c176c15fb1a838763b77c1c06994adba16654b92a666e1b/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:de6fbc071ecc4fe6e354a7939202191c1f1abffe37fbce9b08e7e9a5b93eba3d", size = 3225051 }, - { url = "https://files.pythonhosted.org/packages/b7/0d/a5d703214fe49d261b4b8f0a64140a4dc1f88560724a38ad937120b899ad/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db9e87f6ea4b0ce99b2651203480585fd9e8dd0dd122a19e46836e93e3a1b749", size = 2870421 }, - { url = "https://files.pythonhosted.org/packages/ac/af/41d79cb87eae99c0348e8f1fb3dbed9e40a6f63548b216e99f4d1165fa5c/grpcio_tools-1.67.1-cp312-cp312-win32.whl", hash = "sha256:6a595a872fb720dde924c4e8200f41d5418dd6baab8cc1a3c1e540f8f4596351", size = 940542 }, - { url = "https://files.pythonhosted.org/packages/66/e5/096e12f5319835aa2bcb746d49ae62220bb48313ca649e89bdbef605c11d/grpcio_tools-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:92eebb9b31031604ae97ea7657ae2e43149b0394af7117ad7e15894b6cc136dc", size = 1090425 }, - { url = "https://files.pythonhosted.org/packages/62/b3/91c88440c978740752d39f1abae83f21408048b98b93652ebd84f974ad3d/grpcio_tools-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:9a3b9510cc87b6458b05ad49a6dee38df6af37f9ee6aa027aa086537798c3d4a", size = 2307453 }, - { url = "https://files.pythonhosted.org/packages/05/33/faf3330825463c0409fa3891bc1459bf86a00055b19790211365279538d7/grpcio_tools-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e4c9b9fa9b905f15d414cb7bd007ba7499f8907bdd21231ab287a86b27da81a", size = 5517975 }, - { url = "https://files.pythonhosted.org/packages/bd/78/461ab34cadbd0b5b9a0b6efedda96b58e0de471e3fa91d8e4a4e31924e1b/grpcio_tools-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:e11a98b41af4bc88b7a738232b8fa0306ad82c79fa5d7090bb607f183a57856f", size = 2281081 }, - { url = "https://files.pythonhosted.org/packages/5f/0c/b30bdbcab1795b12e05adf30c20981c14f66198e22044edb15b3c1d9f0bc/grpcio_tools-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de0fcfe61c26679d64b1710746f2891f359593f76894fcf492c37148d5694f00", size = 2616929 }, - { url = "https://files.pythonhosted.org/packages/d3/c2/a77ca68ae768f8d5f1d070ea4afc42fda40401083e7c4f5c08211e84de38/grpcio_tools-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae3b3e2ee5aad59dece65a613624c46a84c9582fc3642686537c6dfae8e47dc", size = 2414633 }, - { url = "https://files.pythonhosted.org/packages/39/70/8d7131dccfe4d7b739c96ada7ea9acde631f58f013eae773791fb490a3eb/grpcio_tools-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:9a630f83505b6471a3094a7a372a1240de18d0cd3e64f4fbf46b361bac2be65b", size = 3224328 }, - { url = "https://files.pythonhosted.org/packages/2a/28/2d24b933ccf0d6877035aa3d5f8b64aad18c953657dd43c682b5701dc127/grpcio_tools-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d85a1fcbacd3e08dc2b3d1d46b749351a9a50899fa35cf2ff040e1faf7d405ad", size = 2869640 }, - { url = "https://files.pythonhosted.org/packages/37/77/ddd2b4cc896639fb0f85fc21d5684f25080ee28845c5a4031e3dd65fdc92/grpcio_tools-1.67.1-cp313-cp313-win32.whl", hash = "sha256:778470f025f25a1fca5a48c93c0a18af395b46b12dd8df7fca63736b85181f41", size = 939997 }, - { url = "https://files.pythonhosted.org/packages/96/d0/f0855a0ccb26ffeb41e6db68b5cbb25d7e9ba1f8f19151eef36210e64efc/grpcio_tools-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:6961da86e9856b4ddee0bf51ef6636b4bf9c29c0715aa71f3c8f027c45d42654", size = 1089819 }, +sdist = { url = "https://files.pythonhosted.org/packages/64/ec/1c25136ca1697eaa09a02effe3e74959fd9fb6aba9960d7340dd6341c5ce/grpcio_tools-1.69.0.tar.gz", hash = "sha256:3e1a98f4d9decb84979e1ddd3deb09c0a33a84b6e3c0776d5bde4097e3ab66dd", size = 5323319 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/00/90/7df7326552fec627adcf3880cf13e9a5b23c090bbcedba367f64fa2bb54b/grpcio_tools-1.69.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:8c210630faa581c3bd08953dac4ad21a7f49862f3b92d69686e9b436d2f1265d", size = 2388795 }, + { url = "https://files.pythonhosted.org/packages/e2/03/6ccaa58b3ca1734d0868a389148e22ac15248a9be4c223805339f7904e31/grpcio_tools-1.69.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:09b66ea279fcdaebae4ec34b1baf7577af3b14322738aa980c1c33cfea71f7d7", size = 5703156 }, + { url = "https://files.pythonhosted.org/packages/c9/f6/162b456684d2444b43e45ace4e889087301e5890bbfd16ee6b2aedf36219/grpcio_tools-1.69.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:be94a4bfa56d356aae242cc54072c9ccc2704b659eaae2fd599a94afebf791ce", size = 2350725 }, + { url = "https://files.pythonhosted.org/packages/db/3a/2e83fea8c90b9902d68964491d014d688177a6ad0303dbbe6c2c16f25da6/grpcio_tools-1.69.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:28778debad73a8c8e0a0e07e6a2f76eecce43adbc205d17dd244d2d58bb0f0aa", size = 2727230 }, + { url = "https://files.pythonhosted.org/packages/63/06/be27b8f1811ff4cc556bdec64a9004755a929df035dc606466a75c9ac0fa/grpcio_tools-1.69.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:449308d93e4c97ae3a4503510c6d64978748ff5e21429c85da14fdc783c0f498", size = 2472752 }, + { url = "https://files.pythonhosted.org/packages/a3/43/f94578afa1535287b7b0ba39eeb23b2b8304a2a5b8e325ed7079d2ad9cba/grpcio_tools-1.69.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b9343651e73bc6e0df6bb518c2638bf9cc2194b50d060cdbcf1b2121cd4e4ae3", size = 3344074 }, + { url = "https://files.pythonhosted.org/packages/13/d1/5f9030cbb6195f3bb182e740f349cdaa71d9c38c1b2572f401270709d7d2/grpcio_tools-1.69.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f08b063612553e726e328aef3a27adfaea8d92712b229012afc54d59da88a02", size = 2953778 }, + { url = "https://files.pythonhosted.org/packages/0c/cb/4812660e150d197de81296fa04ed6ad012d1aeac23bbe21be5f51493f455/grpcio_tools-1.69.0-cp310-cp310-win32.whl", hash = "sha256:599ffd39525e7bbb6412a63e56a2e6c1af8f3493fe4305260efd4a11d064cce0", size = 957556 }, + { url = "https://files.pythonhosted.org/packages/4e/c7/c7d5f5418909764e63208b9f76812db3287ece4f79500e815178194e1db9/grpcio_tools-1.69.0-cp310-cp310-win_amd64.whl", hash = "sha256:02f92e3c2bae67ece818787f8d3d89df0fa1e5e6bbb7c1493824fd5dfad886dd", size = 1114783 }, + { url = "https://files.pythonhosted.org/packages/7e/f4/575f536bada8d8f5f8943c317ae28faafe7b4aaf95ef84a599f4f3e67db3/grpcio_tools-1.69.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c18df5d1c8e163a29863583ec51237d08d7059ef8d4f7661ee6d6363d3e38fe3", size = 2388772 }, + { url = "https://files.pythonhosted.org/packages/87/94/1157342b046f51c4d076f21ef76da6d89323929b7e870389204fd49e3f09/grpcio_tools-1.69.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:37876ae49235ef2e61e5059faf45dc5e7142ca54ae61aec378bb9483e0cd7e95", size = 5726348 }, + { url = "https://files.pythonhosted.org/packages/36/5c/cfd9160ef1867e025844b2695d436bb953c2d5f9c20eaaa7da6fd739ab0c/grpcio_tools-1.69.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:33120920e29959eaa37a1268c6a22af243d086b1a5e5222b4203e29560ece9ce", size = 2350857 }, + { url = "https://files.pythonhosted.org/packages/61/70/10614b8bc39f06548a0586fdd5d97843da4789965e758fba87726bde8c2f/grpcio_tools-1.69.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:788bb3ecd1b44664d829d319b3c1ebc15c7d7b5e7d1f22706ab57d6acd2c6301", size = 2727157 }, + { url = "https://files.pythonhosted.org/packages/37/fb/33faedb3e991dceb7a2bf802d3875bff7d6a6b6a80d314197adc73739cae/grpcio_tools-1.69.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f453b11a112e3774c8957ec2570669f3da1f7fbc8ee242482c38981496e88da2", size = 2472882 }, + { url = "https://files.pythonhosted.org/packages/41/f7/abddc158919a982f6b8e61d4a5c72569b2963304c162c3ca53c6c14d23ee/grpcio_tools-1.69.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7e5c5dc2b656755cb58b11a7e87b65258a4a8eaff01b6c30ffcb230dd447c03d", size = 3343987 }, + { url = "https://files.pythonhosted.org/packages/ba/46/e7219456aefe29137728246a67199fcbfdaa99ede93d2045a6406f0e4c0b/grpcio_tools-1.69.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8eabf0a7a98c14322bc74f9910c96f98feebe311e085624b2d022924d4f652ca", size = 2953659 }, + { url = "https://files.pythonhosted.org/packages/74/be/262c5d2b681930f8c58012500741fe06cb40a770c9d395650efe9042467f/grpcio_tools-1.69.0-cp311-cp311-win32.whl", hash = "sha256:ad567bea43d018c2215e1db10316eda94ca19229a834a3221c15d132d24c1b8a", size = 957447 }, + { url = "https://files.pythonhosted.org/packages/8e/55/68153acca126dced35f888e708a65169df8fa8a4d5f0e78166a395e3fa9c/grpcio_tools-1.69.0-cp311-cp311-win_amd64.whl", hash = "sha256:3d64e801586dbea3530f245d48b9ed031738cc3eb099d5ce2fdb1b3dc2e1fb20", size = 1114753 }, + { url = "https://files.pythonhosted.org/packages/5b/f6/9cd1aa47556664564b873cd187d8dec978ff2f4a539d8c6d5d2f418d3d36/grpcio_tools-1.69.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8ef8efe8beac4cc1e30d41893e4096ca2601da61001897bd17441645de2d4d3c", size = 2388440 }, + { url = "https://files.pythonhosted.org/packages/62/37/0bcd8431e44b38f648f70368dd60542d10ffaffa109563349ee635013e10/grpcio_tools-1.69.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:a00e87a0c5a294028115a098819899b08dd18449df5b2aac4a2b87ba865e8681", size = 5726135 }, + { url = "https://files.pythonhosted.org/packages/8b/f5/2ec994bbf522a231ce54c41a2d3621e77bece1240aafe31f12804052af0f/grpcio_tools-1.69.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:7722700346d5b223159532e046e51f2ff743ed4342e5fe3e0457120a4199015e", size = 2350247 }, + { url = "https://files.pythonhosted.org/packages/a9/29/9ebf54315a499a766e4c3bd53124267491162e9049c2d9ed45f43222b98f/grpcio_tools-1.69.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a934116fdf202cb675246056ee54645c743e2240632f86a37e52f91a405c7143", size = 2727994 }, + { url = "https://files.pythonhosted.org/packages/f0/2a/1a031018660b5d95c1a4c587a0babd0d28f0aa0c9a40dbca330567049a3f/grpcio_tools-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e6a6d44359ca836acfbc58103daf94b3bb8ac919d659bb348dcd7fbecedc293", size = 2472625 }, + { url = "https://files.pythonhosted.org/packages/74/bf/76d24078e1c76976a10760c3193b6c62685a7aed64b1cb0d8242afa16f1d/grpcio_tools-1.69.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e27662c0597fd1ab5399a583d358b5203edcb6fc2b29d6245099dfacd51a6ddc", size = 3344290 }, + { url = "https://files.pythonhosted.org/packages/f1/f7/4ab645e4955ca1e5240b0bbd557662cec4838f0e21e072ff40f4e191b48d/grpcio_tools-1.69.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7bbb2b2fb81d95bcdd1d8331defb5f5dc256dbe423bb98b682cf129cdd432366", size = 2953592 }, + { url = "https://files.pythonhosted.org/packages/8f/32/57e67b126f209f289fc32009309d155b8dbe9ac760c32733746e4dda7b51/grpcio_tools-1.69.0-cp312-cp312-win32.whl", hash = "sha256:e11accd10cf4af5031ac86c45f1a13fb08f55e005cea070917c12e78fe6d2aa2", size = 957042 }, + { url = "https://files.pythonhosted.org/packages/19/64/7bfcb4e50a0ce87690c24696cd666f528e672119966abead09ae65a2e1da/grpcio_tools-1.69.0-cp312-cp312-win_amd64.whl", hash = "sha256:6df4c6ac109af338a8ccde29d184e0b0bdab13d78490cb360ff9b192a1aec7e2", size = 1114248 }, + { url = "https://files.pythonhosted.org/packages/0c/ef/a9867f612e3aa5e69d299e47a72ea8dafa476b1f099462c9a1223cd6a83c/grpcio_tools-1.69.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:8c320c4faa1431f2e1252ef2325a970ac23b2fd04ffef6c12f96dd4552c3445c", size = 2388281 }, + { url = "https://files.pythonhosted.org/packages/4b/53/b2752d8ec338778e48d76845d605a0f8bca9e43a5f09428e5ed1a76e4e1d/grpcio_tools-1.69.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:5f1224596ad74dd14444b20c37122b361c5d203b67e14e018b995f3c5d76eede", size = 5725856 }, + { url = "https://files.pythonhosted.org/packages/83/dd/195d3639634c0c1d1e48b6693c074d66a64f16c748df2f40bcee74aa04e2/grpcio_tools-1.69.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:965a0cf656a113bc32d15ac92ca51ed702a75d5370ae0afbdd36f818533a708a", size = 2350180 }, + { url = "https://files.pythonhosted.org/packages/8c/18/c412884fa0e888d8a271f3e31d23e3765cde0efe2404653ab67971c411c2/grpcio_tools-1.69.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:978835768c11a7f28778b3b7c40f839d8a57f765c315e80c4246c23900d56149", size = 2726724 }, + { url = "https://files.pythonhosted.org/packages/be/c7/dfb59b7e25d760bfdd93f0aef7dd0e2a37f8437ac3017b8b526c68764e2f/grpcio_tools-1.69.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:094c7cec9bd271a32dfb7c620d4a558c63fcb0122fd1651b9ed73d6afd4ae6fe", size = 2472127 }, + { url = "https://files.pythonhosted.org/packages/f2/b6/af4edf0a181fd7b148a83d491f5677d7d1c9f86f03282f8f0209d9dfb793/grpcio_tools-1.69.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:b51bf4981b3d7e47c2569efadff08284787124eb3dea0f63f491d39703231d3c", size = 3344015 }, + { url = "https://files.pythonhosted.org/packages/0a/9f/4c2b5ae642f7d3df73c16df6c7d53e9443cb0e49e1dcf2c8d1a49058e0b5/grpcio_tools-1.69.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ea7aaf0dc1a828e2133357a9e9553fd1bb4e766890d52a506cc132e40632acdc", size = 2952942 }, + { url = "https://files.pythonhosted.org/packages/97/8e/6b707871db5927a17ad7475c070916bff4f32463a51552b424779236ab65/grpcio_tools-1.69.0-cp313-cp313-win32.whl", hash = "sha256:4320f11b79d3a148cc23bad1b81719ce1197808dc2406caa8a8ba0a5cfb0260d", size = 956242 }, + { url = "https://files.pythonhosted.org/packages/27/e2/b419a02b50240143605f77cd50cb07f724caf0fd35a01540a4f044ae9f21/grpcio_tools-1.69.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9bae733654e0eb8ca83aa1d0d6b6c2f4a3525ce70d5ffc07df68d28f6520137", size = 1113616 }, ] [[package]] @@ -2193,6 +2206,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, ] +[[package]] +name = "marshmallow" +version = "3.24.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/3b/1f/52fa79445669322ee42fdd11b591c2e9c8dbab33eaf7059ca881b349ae09/marshmallow-3.24.2.tar.gz", hash = "sha256:0822c3701de396b51d3f8ac97319aea5493998ba4e7d0e4c05f6fce7777bf3a2", size = 176520 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/67/40/7802bb90b1ecbb284ae613da2cfde9ce0177b77d76cbb276acf976296aa8/marshmallow-3.24.2-py3-none-any.whl", hash = "sha256:bf3c56db473bb160e5191f1c5e32e3fc8bfb58998eb2b35d6747de023e31f9e7", size = 49333 }, +] + [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -2240,7 +2265,7 @@ wheels = [ [[package]] name = "mistralai" -version = "1.3.1" +version = "1.2.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "eval-type-backport", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -2250,9 +2275,9 @@ dependencies = [ { name = "python-dateutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-inspect", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/2f/50/59669ee8d21fd27a4f887148b1efb19d9be5ed22ec19c8e6eb842407ac0f/mistralai-1.3.1.tar.gz", hash = "sha256:1c30385656393f993625943045ad20de2aff4c6ab30fc6e8c727d735c22b1c08", size = 133338 } +sdist = { url = "https://files.pythonhosted.org/packages/36/18/53e6bb5c573b130134808236d649748e0280152b4e0c8436f05ff83a86de/mistralai-1.2.6.tar.gz", hash = "sha256:87a2b6fec565e775b0a027474b02be0c219c0a6b787c193ea1c4d12bac08e52e", size = 133264 } wheels = [ - { url = "https://files.pythonhosted.org/packages/1a/b4/a76b6942b78383d5499f776d880a166296542383f6f952feeef96d0ea692/mistralai-1.3.1-py3-none-any.whl", hash = "sha256:35e74feadf835b7d2145095114b9cf3ba86c4cf1044f28f49b02cd6ddd0a5733", size = 261271 }, + { url = "https://files.pythonhosted.org/packages/22/0e/e16e6fd06f5a6345a1fde3a75653769f46a04f92f10db3bb3028b88eba16/mistralai-1.2.6-py3-none-any.whl", hash = "sha256:d9db22ca3a0e029dc2bf8e9380390168440ae4c19d21d212f53ff0d0bd917447", size = 261307 }, ] [[package]] @@ -2886,7 +2911,7 @@ wheels = [ [[package]] name = "openai" -version = "1.59.3" +version = "1.59.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -2898,9 +2923,9 @@ dependencies = [ { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/73/d0/def3c7620e1cb446947f098aeac9d88fc826b1760d66da279e4712d37666/openai-1.59.3.tar.gz", hash = "sha256:7f7fff9d8729968588edf1524e73266e8593bb6cab09298340efb755755bb66f", size = 344192 } +sdist = { url = "https://files.pythonhosted.org/packages/7c/b3/a99ff4f8034383147f853200ff5f6df63a8407a0061d6b3ff47914b94f4c/openai-1.59.5.tar.gz", hash = "sha256:9886e77c02dad9dc6a7b67a11ab372a56842a9b5d376aa476672175ab10e83a0", size = 344773 } wheels = [ - { url = "https://files.pythonhosted.org/packages/c7/26/0e0fb582bcb2a7cb6802447a749a2fc938fe4b82324097abccb86abfd5d1/openai-1.59.3-py3-none-any.whl", hash = "sha256:b041887a0d8f3e70d1fc6ffbb2bf7661c3b9a2f3e806c04bf42f572b9ac7bc37", size = 454793 }, + { url = "https://files.pythonhosted.org/packages/4b/a2/a64f495c016234ca4269005b19eb9193a925dcad01af95eb8fea3de4ee9c/openai-1.59.5-py3-none-any.whl", hash = "sha256:e646b44856b0dda9345d3c43639e056334d792d1690e99690313c0ef7ca4d8cc", size = 454815 }, ] [package.optional-dependencies] @@ -3096,58 +3121,58 @@ wheels = [ [[package]] name = "orjson" -version = "3.10.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/45/0b/8c7eaf1e2152f1e0fb28ae7b22e2b35a6b1992953a1ebe0371ba4d41d3ad/orjson-3.10.13.tar.gz", hash = "sha256:eb9bfb14ab8f68d9d9492d4817ae497788a15fd7da72e14dfabc289c3bb088ec", size = 5438389 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/c4/67206a3cd1b677e2dc8d0de102bebc993ce083548542461e9fa397ce3e7c/orjson-3.10.13-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:1232c5e873a4d1638ef957c5564b4b0d6f2a6ab9e207a9b3de9de05a09d1d920", size = 248733 }, - { url = "https://files.pythonhosted.org/packages/9f/c7/49202bcefb75c614d8f221845dd185d4e4dab1aace9a09e99a840dd22abb/orjson-3.10.13-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d26a0eca3035619fa366cbaf49af704c7cb1d4a0e6c79eced9f6a3f2437964b6", size = 136954 }, - { url = "https://files.pythonhosted.org/packages/87/6c/21518e60589c27cc4bc76156d1a0980fe2be7f5419f5269e800e2e5902bb/orjson-3.10.13-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d4b6acd7c9c829895e50d385a357d4b8c3fafc19c5989da2bae11783b0fd4977", size = 149101 }, - { url = "https://files.pythonhosted.org/packages/e3/88/5eac5856b28df0273ac07187cd20a0e6108799d9f5f3382e2dd1398ec1b3/orjson-3.10.13-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1884e53c6818686891cc6fc5a3a2540f2f35e8c76eac8dc3b40480fb59660b00", size = 140445 }, - { url = "https://files.pythonhosted.org/packages/a9/66/a6455588709b6d0cb4ebc95bc775c19c548d1d1e354bd10ad018123698a2/orjson-3.10.13-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6a428afb5720f12892f64920acd2eeb4d996595bf168a26dd9190115dbf1130d", size = 156532 }, - { url = "https://files.pythonhosted.org/packages/c2/41/58f73d6656f1c9d6e736549f36066ce16ba91e33a639c8cca278af09baf3/orjson-3.10.13-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba5b13b8739ce5b630c65cb1c85aedbd257bcc2b9c256b06ab2605209af75a2e", size = 131261 }, - { url = "https://files.pythonhosted.org/packages/c9/7e/81ca17c438733741265e8ebfa3e5436aa4e61332f91ebdc11eff27c7b152/orjson-3.10.13-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:cab83e67f6aabda1b45882254b2598b48b80ecc112968fc6483fa6dae609e9f0", size = 139822 }, - { url = "https://files.pythonhosted.org/packages/be/fc/b1d72a5f431fc5ae9edfa5bb41fb3b5e9532a4181c5268e67bc2717217bf/orjson-3.10.13-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:62c3cc00c7e776c71c6b7b9c48c5d2701d4c04e7d1d7cdee3572998ee6dc57cc", size = 131901 }, - { url = "https://files.pythonhosted.org/packages/31/f6/8cdcd06e0d4ee37eba1c7a6cd2c5a8798a3a533f9b17b5e48a2a7dcdf6c9/orjson-3.10.13-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:dc03db4922e75bbc870b03fc49734cefbd50fe975e0878327d200022210b82d8", size = 415733 }, - { url = "https://files.pythonhosted.org/packages/f1/37/0aec8417b5a18136651d57af7955a5991a80abca6356cd4dd04a869ee8e6/orjson-3.10.13-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:22f1c9a30b43d14a041a6ea190d9eca8a6b80c4beb0e8b67602c82d30d6eec3e", size = 142454 }, - { url = "https://files.pythonhosted.org/packages/b7/06/679318d8da3ce897b1d0518073abe6b762e7994b4f765b959b39a7d909a4/orjson-3.10.13-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:b42f56821c29e697c68d7d421410d7c1d8f064ae288b525af6a50cf99a4b1200", size = 130672 }, - { url = "https://files.pythonhosted.org/packages/90/e4/3d0018b3aee93385393b37af000214b18c6873bb0d0097ba1355b7cb23d2/orjson-3.10.13-cp310-cp310-win32.whl", hash = "sha256:0dbf3b97e52e093d7c3e93eb5eb5b31dc7535b33c2ad56872c83f0160f943487", size = 143675 }, - { url = "https://files.pythonhosted.org/packages/30/f1/3608a164a4fea07b795ace71862375e2c1686537d8f907d4c9f6f1d63008/orjson-3.10.13-cp310-cp310-win_amd64.whl", hash = "sha256:46c249b4e934453be4ff2e518cd1adcd90467da7391c7a79eaf2fbb79c51e8c7", size = 135084 }, - { url = "https://files.pythonhosted.org/packages/01/44/7a047e47779953e3f657a612ad36f71a0bca02cf57ff490c427e22b01833/orjson-3.10.13-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a36c0d48d2f084c800763473020a12976996f1109e2fcb66cfea442fdf88047f", size = 248732 }, - { url = "https://files.pythonhosted.org/packages/d6/e9/54976977aaacc5030fdd8012479638bb8d4e2a16519b516ac2bd03a48eab/orjson-3.10.13-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0065896f85d9497990731dfd4a9991a45b0a524baec42ef0a63c34630ee26fd6", size = 136954 }, - { url = "https://files.pythonhosted.org/packages/7f/a7/663fb04e031d5c80a348aeb7271c6042d13f80393c4951b8801a703b89c0/orjson-3.10.13-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:92b4ec30d6025a9dcdfe0df77063cbce238c08d0404471ed7a79f309364a3d19", size = 149101 }, - { url = "https://files.pythonhosted.org/packages/f9/f1/5f2a4bf7525ef4acf48902d2df2bcc1c5aa38f6cc17ee0729a1d3e110ddb/orjson-3.10.13-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a94542d12271c30044dadad1125ee060e7a2048b6c7034e432e116077e1d13d2", size = 140445 }, - { url = "https://files.pythonhosted.org/packages/12/d3/e68afa1db9860880e59260348b54c0518d8dfe2297e932f8e333ace878fa/orjson-3.10.13-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3723e137772639af8adb68230f2aa4bcb27c48b3335b1b1e2d49328fed5e244c", size = 156530 }, - { url = "https://files.pythonhosted.org/packages/77/ee/492b198c77b9985ae28e0c6b8092c2994cd18d6be40dc7cb7f9a385b7096/orjson-3.10.13-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f00c7fb18843bad2ac42dc1ce6dd214a083c53f1e324a0fd1c8137c6436269b", size = 131260 }, - { url = "https://files.pythonhosted.org/packages/57/d2/5167cc1ccbe56bacdd9fc79e6a3276cba6aa90057305e8485db58b8250c4/orjson-3.10.13-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:0e2759d3172300b2f892dee85500b22fca5ac49e0c42cfff101aaf9c12ac9617", size = 139821 }, - { url = "https://files.pythonhosted.org/packages/74/f0/c1cf568e0f90d812e00c77da2db04a13e94afe639665b9a09c271456dc41/orjson-3.10.13-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:ee948c6c01f6b337589c88f8e0bb11e78d32a15848b8b53d3f3b6fea48842c12", size = 131904 }, - { url = "https://files.pythonhosted.org/packages/55/7d/a611542afbbacca4693a2319744944134df62957a1f206303d5b3160e349/orjson-3.10.13-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:aa6fe68f0981fba0d4bf9cdc666d297a7cdba0f1b380dcd075a9a3dd5649a69e", size = 415733 }, - { url = "https://files.pythonhosted.org/packages/64/3f/e8182716695cd8d5ebec49d283645b8c7b1de7ed1c27db2891b6957e71f6/orjson-3.10.13-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:dbcd7aad6bcff258f6896abfbc177d54d9b18149c4c561114f47ebfe74ae6bfd", size = 142456 }, - { url = "https://files.pythonhosted.org/packages/dc/10/e4b40f15be7e4e991737d77062399c7f67da9b7e3bc28bbcb25de1717df3/orjson-3.10.13-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:2149e2fcd084c3fd584881c7f9d7f9e5ad1e2e006609d8b80649655e0d52cd02", size = 130676 }, - { url = "https://files.pythonhosted.org/packages/ad/b1/8b9fb36d470fe8ff99727972c77846673ebc962cb09a5af578804f9f2408/orjson-3.10.13-cp311-cp311-win32.whl", hash = "sha256:89367767ed27b33c25c026696507c76e3d01958406f51d3a2239fe9e91959df2", size = 143672 }, - { url = "https://files.pythonhosted.org/packages/b5/15/90b3711f40d27aff80dd42c1eec2f0ed704a1fa47eef7120350e2797892d/orjson-3.10.13-cp311-cp311-win_amd64.whl", hash = "sha256:dca1d20f1af0daff511f6e26a27354a424f0b5cf00e04280279316df0f604a6f", size = 135082 }, - { url = "https://files.pythonhosted.org/packages/35/84/adf8842cf36904e6200acff76156862d48d39705054c1e7c5fa98fe14417/orjson-3.10.13-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:a3614b00621c77f3f6487792238f9ed1dd8a42f2ec0e6540ee34c2d4e6db813a", size = 248778 }, - { url = "https://files.pythonhosted.org/packages/69/2f/22ac0c5f46748e9810287a5abaeabdd67f1120a74140db7d529582c92342/orjson-3.10.13-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9c976bad3996aa027cd3aef78aa57873f3c959b6c38719de9724b71bdc7bd14b", size = 136759 }, - { url = "https://files.pythonhosted.org/packages/39/67/6f05de77dd383cb623e2807bceae13f136e9f179cd32633b7a27454e953f/orjson-3.10.13-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:5f74d878d1efb97a930b8a9f9898890067707d683eb5c7e20730030ecb3fb930", size = 149123 }, - { url = "https://files.pythonhosted.org/packages/f8/5c/b5e144e9adbb1dc7d1fdf54af9510756d09b65081806f905d300a926a755/orjson-3.10.13-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:33ef84f7e9513fb13b3999c2a64b9ca9c8143f3da9722fbf9c9ce51ce0d8076e", size = 140557 }, - { url = "https://files.pythonhosted.org/packages/91/fd/7bdbc0aa374d49cdb917ee51c80851c99889494be81d5e7ec9f5f9cbe149/orjson-3.10.13-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:dd2bcde107221bb9c2fa0c4aaba735a537225104173d7e19cf73f70b3126c993", size = 156626 }, - { url = "https://files.pythonhosted.org/packages/48/90/e583d6e29937ec30a164f1d86a0439c1a2477b5aae9f55d94b37a4f5b5f0/orjson-3.10.13-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:064b9dbb0217fd64a8d016a8929f2fae6f3312d55ab3036b00b1d17399ab2f3e", size = 131551 }, - { url = "https://files.pythonhosted.org/packages/47/0b/838c00ec7f048527aa0382299cd178bbe07c2cb1024b3111883e85d56d1f/orjson-3.10.13-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:c0044b0b8c85a565e7c3ce0a72acc5d35cda60793edf871ed94711e712cb637d", size = 139790 }, - { url = "https://files.pythonhosted.org/packages/ac/90/df06ac390f319a61d55a7a4efacb5d7082859f6ea33f0fdd5181ad0dde0c/orjson-3.10.13-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:7184f608ad563032e398f311910bc536e62b9fbdca2041be889afcbc39500de8", size = 131717 }, - { url = "https://files.pythonhosted.org/packages/ea/68/eafb5e2fc84aafccfbd0e9e0552ff297ef5f9b23c7f2600cc374095a50de/orjson-3.10.13-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:d36f689e7e1b9b6fb39dbdebc16a6f07cbe994d3644fb1c22953020fc575935f", size = 415690 }, - { url = "https://files.pythonhosted.org/packages/b8/cf/aa93b48801b2e42da223ef5a99b3e4970b02e7abea8509dd2a6a083e27fa/orjson-3.10.13-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:54433e421618cd5873e51c0e9d0b9fb35f7bf76eb31c8eab20b3595bb713cd3d", size = 142396 }, - { url = "https://files.pythonhosted.org/packages/8b/50/fb1a7060b79231c60a688037c2c8e9fe289b5a4378ec1f32cf8d33d9adf8/orjson-3.10.13-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:e1ba0c5857dd743438acecc1cd0e1adf83f0a81fee558e32b2b36f89e40cee8b", size = 130842 }, - { url = "https://files.pythonhosted.org/packages/94/e6/44067052e28a13176da874ca53419b43cf0f6f01f4bf0539f2f70d8eacf6/orjson-3.10.13-cp312-cp312-win32.whl", hash = "sha256:a42b9fe4b0114b51eb5cdf9887d8c94447bc59df6dbb9c5884434eab947888d8", size = 143773 }, - { url = "https://files.pythonhosted.org/packages/f2/7d/510939d1b7f8ba387849e83666e898f214f38baa46c5efde94561453974d/orjson-3.10.13-cp312-cp312-win_amd64.whl", hash = "sha256:3a7df63076435f39ec024bdfeb4c9767ebe7b49abc4949068d61cf4857fa6d6c", size = 135234 }, - { url = "https://files.pythonhosted.org/packages/ef/42/482fced9a135c798f31e1088f608fa16735fdc484eb8ffdd29aa32d4e842/orjson-3.10.13-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:2cdaf8b028a976ebab837a2c27b82810f7fc76ed9fb243755ba650cc83d07730", size = 248726 }, - { url = "https://files.pythonhosted.org/packages/00/e7/6345653906ee6d2d6eabb767cdc4482c7809572dbda59224f40e48931efa/orjson-3.10.13-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:48a946796e390cbb803e069472de37f192b7a80f4ac82e16d6eb9909d9e39d56", size = 126032 }, - { url = "https://files.pythonhosted.org/packages/ad/b8/0d2a2c739458ff7f9917a132225365d72d18f4b65c50cb8ebb5afb6fe184/orjson-3.10.13-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d64f1db5ecbc21eb83097e5236d6ab7e86092c1cd4c216c02533332951afc", size = 131547 }, - { url = "https://files.pythonhosted.org/packages/8d/ac/a1dc389cf364d576cf587a6f78dac6c905c5cac31b9dbd063bbb24335bf7/orjson-3.10.13-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:711878da48f89df194edd2ba603ad42e7afed74abcd2bac164685e7ec15f96de", size = 131682 }, - { url = "https://files.pythonhosted.org/packages/43/6c/debab76b830aba6449ec8a75ac77edebb0e7decff63eb3ecfb2cf6340a2e/orjson-3.10.13-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:cf16f06cb77ce8baf844bc222dbcb03838f61d0abda2c3341400c2b7604e436e", size = 415621 }, - { url = "https://files.pythonhosted.org/packages/c2/32/106e605db5369a6717036065e2b41ac52bd0d2712962edb3e026a452dc07/orjson-3.10.13-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:8257c3fb8dd7b0b446b5e87bf85a28e4071ac50f8c04b6ce2d38cb4abd7dff57", size = 142388 }, - { url = "https://files.pythonhosted.org/packages/a3/02/6b2103898d60c2565bf97abffdf3a4cf338920b9feb55eec1fd791ab10ee/orjson-3.10.13-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:d9c3a87abe6f849a4a7ac8a8a1dede6320a4303d5304006b90da7a3cd2b70d2c", size = 130825 }, - { url = "https://files.pythonhosted.org/packages/87/7c/db115e2380435da569732999d5c4c9b9868efe72e063493cb73c36bb649a/orjson-3.10.13-cp313-cp313-win32.whl", hash = "sha256:527afb6ddb0fa3fe02f5d9fba4920d9d95da58917826a9be93e0242da8abe94a", size = 143723 }, - { url = "https://files.pythonhosted.org/packages/cc/5e/c2b74a0b38ec561a322d8946663924556c1f967df2eefe1b9e0b98a33950/orjson-3.10.13-cp313-cp313-win_amd64.whl", hash = "sha256:b5f7c298d4b935b222f52d6c7f2ba5eafb59d690d9a3840b7b5c5cda97f6ec5c", size = 134968 }, +version = "3.10.14" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/92/f7/3219b56f47b4f5e864fb11cdf4ac0aaa3de608730ad2dc4c6e16382f35ec/orjson-3.10.14.tar.gz", hash = "sha256:cf31f6f071a6b8e7aa1ead1fa27b935b48d00fbfa6a28ce856cfff2d5dd68eed", size = 5282116 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b3/62/64348b8b29a14c7342f6aa45c8be0a87fdda2ce7716bc123717376537077/orjson-3.10.14-cp310-cp310-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:849ea7845a55f09965826e816cdc7689d6cf74fe9223d79d758c714af955bcb6", size = 249439 }, + { url = "https://files.pythonhosted.org/packages/9f/51/48f4dfbca7b4db630316b170db4a150a33cd405650258bd62a2d619b43b4/orjson-3.10.14-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b5947b139dfa33f72eecc63f17e45230a97e741942955a6c9e650069305eb73d", size = 135811 }, + { url = "https://files.pythonhosted.org/packages/a1/1c/e18770843e6d045605c8e00a1be801da5668fa934b323b0492a49c9dee4f/orjson-3.10.14-cp310-cp310-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:cde6d76910d3179dae70f164466692f4ea36da124d6fb1a61399ca589e81d69a", size = 150154 }, + { url = "https://files.pythonhosted.org/packages/51/1e/3817dc79164f1fc17fc53102f74f62d31f5f4ec042abdd24d94c5e06e51c/orjson-3.10.14-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c6dfbaeb7afa77ca608a50e2770a0461177b63a99520d4928e27591b142c74b1", size = 139740 }, + { url = "https://files.pythonhosted.org/packages/ff/fc/fbf9e25448f7a2d67c1a2b6dad78a9340666bf9fda3339ff59b1e93f0b6f/orjson-3.10.14-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fa45e489ef80f28ff0e5ba0a72812b8cfc7c1ef8b46a694723807d1b07c89ebb", size = 154479 }, + { url = "https://files.pythonhosted.org/packages/d4/df/c8b7ea21ff658f6a9a26d562055631c01d445bda5eb613c02c7d0934607d/orjson-3.10.14-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4f5007abfdbb1d866e2aa8990bd1c465f0f6da71d19e695fc278282be12cffa5", size = 130414 }, + { url = "https://files.pythonhosted.org/packages/df/f7/e29c2d42bef8fbf696a5e54e6339b0b9ea5179326950fee6ae80acf59d09/orjson-3.10.14-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:1b49e2af011c84c3f2d541bb5cd1e3c7c2df672223e7e3ea608f09cf295e5f8a", size = 138545 }, + { url = "https://files.pythonhosted.org/packages/8e/97/afdf2908fe8eaeecb29e97fa82dc934f275acf330e5271def0b8fbac5478/orjson-3.10.14-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:164ac155109226b3a2606ee6dda899ccfbe6e7e18b5bdc3fbc00f79cc074157d", size = 130952 }, + { url = "https://files.pythonhosted.org/packages/4a/dd/04e01c1305694f47e9794c60ec7cece02e55fa9d57c5d72081eaaa62ad1d/orjson-3.10.14-cp310-cp310-musllinux_1_2_armv7l.whl", hash = "sha256:6b1225024cf0ef5d15934b5ffe9baf860fe8bc68a796513f5ea4f5056de30bca", size = 414673 }, + { url = "https://files.pythonhosted.org/packages/fa/12/28c4d5f6a395ac9693b250f0662366968c47fc99c8f3cd803a65b1f5ba46/orjson-3.10.14-cp310-cp310-musllinux_1_2_i686.whl", hash = "sha256:d6546e8073dc382e60fcae4a001a5a1bc46da5eab4a4878acc2d12072d6166d5", size = 141002 }, + { url = "https://files.pythonhosted.org/packages/21/f6/357cb167c2d2fd9542251cfd9f68681b67ed4dcdac82aa6ee2f4f3ab952e/orjson-3.10.14-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:9f1d2942605c894162252d6259b0121bf1cb493071a1ea8cb35d79cb3e6ac5bc", size = 129626 }, + { url = "https://files.pythonhosted.org/packages/df/07/d9062353500df9db8bfa7c6a5982687c97d0b69a5b158c4166d407ac94e2/orjson-3.10.14-cp310-cp310-win32.whl", hash = "sha256:397083806abd51cf2b3bbbf6c347575374d160331a2d33c5823e22249ad3118b", size = 142429 }, + { url = "https://files.pythonhosted.org/packages/50/ba/6ba2bf69ac0526d143aebe78bc39e6e5fbb51d5336fbc5efb9aab6687cd9/orjson-3.10.14-cp310-cp310-win_amd64.whl", hash = "sha256:fa18f949d3183a8d468367056be989666ac2bef3a72eece0bade9cdb733b3c28", size = 133512 }, + { url = "https://files.pythonhosted.org/packages/bf/18/26721760368e12b691fb6811692ed21ae5275ea918db409ba26866cacbe8/orjson-3.10.14-cp311-cp311-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:f506fd666dd1ecd15a832bebc66c4df45c1902fd47526292836c339f7ba665a9", size = 249437 }, + { url = "https://files.pythonhosted.org/packages/d5/5b/2adfe7cc301edeb3bffc1942956659c19ec00d51a21c53c17c0767bebf47/orjson-3.10.14-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:efe5fd254cfb0eeee13b8ef7ecb20f5d5a56ddda8a587f3852ab2cedfefdb5f6", size = 135812 }, + { url = "https://files.pythonhosted.org/packages/8a/68/07df7787fd9ff6dba815b2d793eec5e039d288fdf150431ed48a660bfcbb/orjson-3.10.14-cp311-cp311-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:4ddc8c866d7467f5ee2991397d2ea94bcf60d0048bdd8ca555740b56f9042725", size = 150153 }, + { url = "https://files.pythonhosted.org/packages/02/71/f68562734461b801b53bacd5365e079dcb3c78656a662f0639494880e522/orjson-3.10.14-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3af8e42ae4363773658b8d578d56dedffb4f05ceeb4d1d4dd3fb504950b45526", size = 139742 }, + { url = "https://files.pythonhosted.org/packages/04/03/1355fb27652582f00d3c62e93a32b982fa42bc31d2e07f0a317867069096/orjson-3.10.14-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:84dd83110503bc10e94322bf3ffab8bc49150176b49b4984dc1cce4c0a993bf9", size = 154479 }, + { url = "https://files.pythonhosted.org/packages/7c/47/1c2a840f27715e8bc2bbafffc851512ede6e53483593eded190919bdcaf4/orjson-3.10.14-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:36f5bfc0399cd4811bf10ec7a759c7ab0cd18080956af8ee138097d5b5296a95", size = 130413 }, + { url = "https://files.pythonhosted.org/packages/dd/b2/5bb51006cbae85b052d1bbee7ff43ae26fa155bb3d31a71b0c07d384d5e3/orjson-3.10.14-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:868943660fb2a1e6b6b965b74430c16a79320b665b28dd4511d15ad5038d37d5", size = 138545 }, + { url = "https://files.pythonhosted.org/packages/79/30/7841a5dd46bb46b8e868791d5469c9d4788d3e26b7e69d40256647997baf/orjson-3.10.14-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:33449c67195969b1a677533dee9d76e006001213a24501333624623e13c7cc8e", size = 130953 }, + { url = "https://files.pythonhosted.org/packages/08/49/720e7c2040c0f1df630a36d83d449bd7e4d4471071d5ece47a4f7211d570/orjson-3.10.14-cp311-cp311-musllinux_1_2_armv7l.whl", hash = "sha256:e4c9f60f9fb0b5be66e416dcd8c9d94c3eabff3801d875bdb1f8ffc12cf86905", size = 414675 }, + { url = "https://files.pythonhosted.org/packages/50/b0/ca7619f34280e7dcbd50dbc9c5fe5200c12cd7269b8858652beb3887483f/orjson-3.10.14-cp311-cp311-musllinux_1_2_i686.whl", hash = "sha256:0de4d6315cfdbd9ec803b945c23b3a68207fd47cbe43626036d97e8e9561a436", size = 141004 }, + { url = "https://files.pythonhosted.org/packages/75/1b/7548e3a711543f438e87a4349e00439ab7f37807942e5659f29363f35765/orjson-3.10.14-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:83adda3db595cb1a7e2237029b3249c85afbe5c747d26b41b802e7482cb3933e", size = 129629 }, + { url = "https://files.pythonhosted.org/packages/b0/1e/4930a6ff46debd6be1ff18e869b7bc43a7ad762c865610b7e745038d6f68/orjson-3.10.14-cp311-cp311-win32.whl", hash = "sha256:998019ef74a4997a9d741b1473533cdb8faa31373afc9849b35129b4b8ec048d", size = 142430 }, + { url = "https://files.pythonhosted.org/packages/28/e0/6cc1cd1dfde36555e81ac869f7847e86bb11c27f97b72fde2f1509b12163/orjson-3.10.14-cp311-cp311-win_amd64.whl", hash = "sha256:9d034abdd36f0f0f2240f91492684e5043d46f290525d1117712d5b8137784eb", size = 133516 }, + { url = "https://files.pythonhosted.org/packages/8c/dc/dc5a882be016ee8688bd867ad3b4e3b2ab039d91383099702301a1adb6ac/orjson-3.10.14-cp312-cp312-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:2ad4b7e367efba6dc3f119c9a0fcd41908b7ec0399a696f3cdea7ec477441b09", size = 249396 }, + { url = "https://files.pythonhosted.org/packages/f0/95/4c23ff5c0505cd687928608e0b7910ccb44ce59490079e1c17b7610aa0d0/orjson-3.10.14-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f496286fc85e93ce0f71cc84fc1c42de2decf1bf494094e188e27a53694777a7", size = 135689 }, + { url = "https://files.pythonhosted.org/packages/ad/39/b4bdd19604dce9d6509c4d86e8e251a1373a24204b4c4169866dcecbe5f5/orjson-3.10.14-cp312-cp312-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:c7f189bbfcded40e41a6969c1068ba305850ba016665be71a217918931416fbf", size = 150136 }, + { url = "https://files.pythonhosted.org/packages/1d/92/7b9bad96353abd3e89947960252dcf1022ce2df7f29056e434de05e18b6d/orjson-3.10.14-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8cc8204f0b75606869c707da331058ddf085de29558b516fc43c73ee5ee2aadb", size = 139766 }, + { url = "https://files.pythonhosted.org/packages/a6/bd/abb13c86540b7a91b40d7d9f8549d03a026bc22d78fa93f71d68b8f4c36e/orjson-3.10.14-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:deaa2899dff7f03ab667e2ec25842d233e2a6a9e333efa484dfe666403f3501c", size = 154533 }, + { url = "https://files.pythonhosted.org/packages/c0/02/0bcb91ec9c7143012359983aca44f567f87df379957cd4af11336217b12f/orjson-3.10.14-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1c3ea52642c9714dc6e56de8a451a066f6d2707d273e07fe8a9cc1ba073813d", size = 130658 }, + { url = "https://files.pythonhosted.org/packages/b4/1e/b304596bb1f800d47d6e92305bd09f0eef693ed4f7b2095db63f9808b229/orjson-3.10.14-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:9d3f9ed72e7458ded9a1fb1b4d4ed4c4fdbaf82030ce3f9274b4dc1bff7ace2b", size = 138546 }, + { url = "https://files.pythonhosted.org/packages/56/c7/65d72b22080186ef618a46afeb9386e20056f3237664090f3a2f8da1cd6d/orjson-3.10.14-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:07520685d408a2aba514c17ccc16199ff2934f9f9e28501e676c557f454a37fe", size = 130774 }, + { url = "https://files.pythonhosted.org/packages/4d/85/1ab35a832f32b37ccd673721e845cf302f23453603112255af611c91d1d1/orjson-3.10.14-cp312-cp312-musllinux_1_2_armv7l.whl", hash = "sha256:76344269b550ea01488d19a2a369ab572c1ac4449a72e9f6ac0d70eb1cbfb953", size = 414649 }, + { url = "https://files.pythonhosted.org/packages/d1/7d/1d6575f779bab8fe698fa6d52e8aa3aa0a9fca4885d0bf6197700455713a/orjson-3.10.14-cp312-cp312-musllinux_1_2_i686.whl", hash = "sha256:e2979d0f2959990620f7e62da6cd954e4620ee815539bc57a8ae46e2dacf90e3", size = 141060 }, + { url = "https://files.pythonhosted.org/packages/f8/26/68513e28b3bd1d7633318ed2818e86d1bfc8b782c87c520c7b363092837f/orjson-3.10.14-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:03f61ca3674555adcb1aa717b9fc87ae936aa7a63f6aba90a474a88701278780", size = 129798 }, + { url = "https://files.pythonhosted.org/packages/44/ca/020fb99c98ff7267ba18ce798ff0c8c3aa97cd949b611fc76cad3c87e534/orjson-3.10.14-cp312-cp312-win32.whl", hash = "sha256:d5075c54edf1d6ad81d4c6523ce54a748ba1208b542e54b97d8a882ecd810fd1", size = 142524 }, + { url = "https://files.pythonhosted.org/packages/70/7f/f2d346819a273653825e7c92dc26418c8da506003c9fc1dfe8157e733b2e/orjson-3.10.14-cp312-cp312-win_amd64.whl", hash = "sha256:175cafd322e458603e8ce73510a068d16b6e6f389c13f69bf16de0e843d7d406", size = 133663 }, + { url = "https://files.pythonhosted.org/packages/46/bb/f1b037d89f580c79eda0940772384cc226a697be1cb4eb94ae4e792aa34c/orjson-3.10.14-cp313-cp313-macosx_10_15_x86_64.macosx_11_0_arm64.macosx_10_15_universal2.whl", hash = "sha256:0905ca08a10f7e0e0c97d11359609300eb1437490a7f32bbaa349de757e2e0c7", size = 249333 }, + { url = "https://files.pythonhosted.org/packages/e4/72/12958a073cace3f8acef0f9a30739d95f46bbb1544126fecad11527d4508/orjson-3.10.14-cp313-cp313-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:92d13292249f9f2a3e418cbc307a9fbbef043c65f4bd8ba1eb620bc2aaba3d15", size = 125038 }, + { url = "https://files.pythonhosted.org/packages/c0/ae/461f78b1c98de1bc034af88bc21c6a792cc63373261fbc10a6ee560814fa/orjson-3.10.14-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90937664e776ad316d64251e2fa2ad69265e4443067668e4727074fe39676414", size = 130604 }, + { url = "https://files.pythonhosted.org/packages/ae/d2/17f50513f56bff7898840fddf7fb88f501305b9b2605d2793ff224789665/orjson-3.10.14-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:9ed3d26c4cb4f6babaf791aa46a029265850e80ec2a566581f5c2ee1a14df4f1", size = 130756 }, + { url = "https://files.pythonhosted.org/packages/fa/bc/673856e4af94c9890dfd8e2054c05dc2ddc16d1728c2aa0c5bd198943105/orjson-3.10.14-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:56ee546c2bbe9599aba78169f99d1dc33301853e897dbaf642d654248280dc6e", size = 414613 }, + { url = "https://files.pythonhosted.org/packages/09/01/08c5b69b0756dd1790fcffa569d6a28dedcd7b97f825e4b46537b788908c/orjson-3.10.14-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:901e826cb2f1bdc1fcef3ef59adf0c451e8f7c0b5deb26c1a933fb66fb505eae", size = 141010 }, + { url = "https://files.pythonhosted.org/packages/5b/98/72883bb6cf88fd364996e62d2026622ca79bfb8dbaf96ccdd2018ada25b1/orjson-3.10.14-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:26336c0d4b2d44636e1e1e6ed1002f03c6aae4a8a9329561c8883f135e9ff010", size = 129732 }, + { url = "https://files.pythonhosted.org/packages/e4/99/347418f7ef56dcb478ba131a6112b8ddd5b747942652b6e77a53155a7e21/orjson-3.10.14-cp313-cp313-win32.whl", hash = "sha256:e2bc525e335a8545c4e48f84dd0328bc46158c9aaeb8a1c2276546e94540ea3d", size = 142504 }, + { url = "https://files.pythonhosted.org/packages/59/ac/5e96cad01083015f7bfdb02ccafa489da8e6caa7f4c519e215f04d2bd856/orjson-3.10.14-cp313-cp313-win_amd64.whl", hash = "sha256:eca04dfd792cedad53dc9a917da1a522486255360cb4e77619343a20d9f35364", size = 133388 }, ] [[package]] @@ -3546,16 +3571,16 @@ wheels = [ [[package]] name = "protobuf" -version = "5.29.2" +version = "5.29.3" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/a5/73/4e6295c1420a9d20c9c351db3a36109b4c9aa601916cb7c6871e3196a1ca/protobuf-5.29.2.tar.gz", hash = "sha256:b2cc8e8bb7c9326996f0e160137b0861f1a82162502658df2951209d0cb0309e", size = 424901 } +sdist = { url = "https://files.pythonhosted.org/packages/f7/d1/e0a911544ca9993e0f17ce6d3cc0932752356c1b0a834397f28e63479344/protobuf-5.29.3.tar.gz", hash = "sha256:5da0f41edaf117bde316404bad1a486cb4ededf8e4a54891296f648e8e076620", size = 424945 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/42/6db5387124708d619ffb990a846fb123bee546f52868039f8fa964c5bc54/protobuf-5.29.2-cp310-abi3-win32.whl", hash = "sha256:c12ba8249f5624300cf51c3d0bfe5be71a60c63e4dcf51ffe9a68771d958c851", size = 422697 }, - { url = "https://files.pythonhosted.org/packages/6c/38/2fcc968b377b531882d6ab2ac99b10ca6d00108394f6ff57c2395fb7baff/protobuf-5.29.2-cp310-abi3-win_amd64.whl", hash = "sha256:842de6d9241134a973aab719ab42b008a18a90f9f07f06ba480df268f86432f9", size = 434495 }, - { url = "https://files.pythonhosted.org/packages/cb/26/41debe0f6615fcb7e97672057524687ed86fcd85e3da3f031c30af8f0c51/protobuf-5.29.2-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a0c53d78383c851bfa97eb42e3703aefdc96d2036a41482ffd55dc5f529466eb", size = 417812 }, - { url = "https://files.pythonhosted.org/packages/e4/20/38fc33b60dcfb380507b99494aebe8c34b68b8ac7d32808c4cebda3f6f6b/protobuf-5.29.2-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:494229ecd8c9009dd71eda5fd57528395d1eacdf307dbece6c12ad0dd09e912e", size = 319562 }, - { url = "https://files.pythonhosted.org/packages/90/4d/c3d61e698e0e41d926dbff6aa4e57428ab1a6fc3b5e1deaa6c9ec0fd45cf/protobuf-5.29.2-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:b6b0d416bbbb9d4fbf9d0561dbfc4e324fd522f61f7af0fe0f282ab67b22477e", size = 319662 }, - { url = "https://files.pythonhosted.org/packages/f3/fd/c7924b4c2a1c61b8f4b64edd7a31ffacf63432135a2606f03a2f0d75a750/protobuf-5.29.2-py3-none-any.whl", hash = "sha256:fde4554c0e578a5a0bcc9a276339594848d1e89f9ea47b4427c80e5d72f90181", size = 172539 }, + { url = "https://files.pythonhosted.org/packages/dc/7a/1e38f3cafa022f477ca0f57a1f49962f21ad25850c3ca0acd3b9d0091518/protobuf-5.29.3-cp310-abi3-win32.whl", hash = "sha256:3ea51771449e1035f26069c4c7fd51fba990d07bc55ba80701c78f886bf9c888", size = 422708 }, + { url = "https://files.pythonhosted.org/packages/61/fa/aae8e10512b83de633f2646506a6d835b151edf4b30d18d73afd01447253/protobuf-5.29.3-cp310-abi3-win_amd64.whl", hash = "sha256:a4fa6f80816a9a0678429e84973f2f98cbc218cca434abe8db2ad0bffc98503a", size = 434508 }, + { url = "https://files.pythonhosted.org/packages/dd/04/3eaedc2ba17a088961d0e3bd396eac764450f431621b58a04ce898acd126/protobuf-5.29.3-cp38-abi3-macosx_10_9_universal2.whl", hash = "sha256:a8434404bbf139aa9e1300dbf989667a83d42ddda9153d8ab76e0d5dcaca484e", size = 417825 }, + { url = "https://files.pythonhosted.org/packages/4f/06/7c467744d23c3979ce250397e26d8ad8eeb2bea7b18ca12ad58313c1b8d5/protobuf-5.29.3-cp38-abi3-manylinux2014_aarch64.whl", hash = "sha256:daaf63f70f25e8689c072cfad4334ca0ac1d1e05a92fc15c54eb9cf23c3efd84", size = 319573 }, + { url = "https://files.pythonhosted.org/packages/a8/45/2ebbde52ad2be18d3675b6bee50e68cd73c9e0654de77d595540b5129df8/protobuf-5.29.3-cp38-abi3-manylinux2014_x86_64.whl", hash = "sha256:c027e08a08be10b67c06bf2370b99c811c466398c357e615ca88c91c07f0910f", size = 319672 }, + { url = "https://files.pythonhosted.org/packages/fd/b2/ab07b09e0f6d143dfb839693aa05765257bceaa13d03bf1a696b78323e7a/protobuf-5.29.3-py3-none-any.whl", hash = "sha256:0a18ed4a24198528f2333802eb075e59dea9d679ab7a6c5efb017a59004d849f", size = 172550 }, ] [[package]] @@ -3738,22 +3763,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/89/bc88a6711935ba795a679ea6ebee07e128050d6382eaa35a0a47c8032bdc/pyasn1_modules-0.4.1-py3-none-any.whl", hash = "sha256:49bfa96b45a292b711e986f222502c1c9a5e1f4e568fc30e2574a6c7d07838fd", size = 181537 }, ] -[[package]] -name = "pyaudio" -version = "0.2.14" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/26/1d/8878c7752febb0f6716a7e1a52cb92ac98871c5aa522cba181878091607c/PyAudio-0.2.14.tar.gz", hash = "sha256:78dfff3879b4994d1f4fc6485646a57755c6ee3c19647a491f790a0895bd2f87", size = 47066 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/90/90/1553487277e6aa25c0b7c2c38709cdd2b49e11c66c0b25c6e8b7b6638c72/PyAudio-0.2.14-cp310-cp310-win32.whl", hash = "sha256:126065b5e82a1c03ba16e7c0404d8f54e17368836e7d2d92427358ad44fefe61", size = 144624 }, - { url = "https://files.pythonhosted.org/packages/27/bc/719d140ee63cf4b0725016531d36743a797ffdbab85e8536922902c9349a/PyAudio-0.2.14-cp310-cp310-win_amd64.whl", hash = "sha256:2a166fc88d435a2779810dd2678354adc33499e9d4d7f937f28b20cc55893e83", size = 164069 }, - { url = "https://files.pythonhosted.org/packages/7b/f0/b0eab89eafa70a86b7b566a4df2f94c7880a2d483aa8de1c77d335335b5b/PyAudio-0.2.14-cp311-cp311-win32.whl", hash = "sha256:506b32a595f8693811682ab4b127602d404df7dfc453b499c91a80d0f7bad289", size = 144624 }, - { url = "https://files.pythonhosted.org/packages/82/d8/f043c854aad450a76e476b0cf9cda1956419e1dacf1062eb9df3c0055abe/PyAudio-0.2.14-cp311-cp311-win_amd64.whl", hash = "sha256:bbeb01d36a2f472ae5ee5e1451cacc42112986abe622f735bb870a5db77cf903", size = 164070 }, - { url = "https://files.pythonhosted.org/packages/8d/45/8d2b76e8f6db783f9326c1305f3f816d4a12c8eda5edc6a2e1d03c097c3b/PyAudio-0.2.14-cp312-cp312-win32.whl", hash = "sha256:5fce4bcdd2e0e8c063d835dbe2860dac46437506af509353c7f8114d4bacbd5b", size = 144750 }, - { url = "https://files.pythonhosted.org/packages/b0/6a/d25812e5f79f06285767ec607b39149d02aa3b31d50c2269768f48768930/PyAudio-0.2.14-cp312-cp312-win_amd64.whl", hash = "sha256:12f2f1ba04e06ff95d80700a78967897a489c05e093e3bffa05a84ed9c0a7fa3", size = 164126 }, - { url = "https://files.pythonhosted.org/packages/3a/77/66cd37111a87c1589b63524f3d3c848011d21ca97828422c7fde7665ff0d/PyAudio-0.2.14-cp313-cp313-win32.whl", hash = "sha256:95328285b4dab57ea8c52a4a996cb52be6d629353315be5bfda403d15932a497", size = 150982 }, - { url = "https://files.pythonhosted.org/packages/a5/8b/7f9a061c1cc2b230f9ac02a6003fcd14c85ce1828013aecbaf45aa988d20/PyAudio-0.2.14-cp313-cp313-win_amd64.whl", hash = "sha256:692d8c1446f52ed2662120bcd9ddcb5aa2b71f38bda31e58b19fb4672fffba69", size = 173655 }, -] - [[package]] name = "pybars4" version = "0.9.13" @@ -3874,22 +3883,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/46/93416fdae86d40879714f72956ac14df9c7b76f7d41a4d68aa9f71a0028b/pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd", size = 29718 }, ] -[[package]] -name = "pydub" -version = "0.25.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fe/9a/e6bca0eed82db26562c73b5076539a4a08d3cffd19c3cc5913a3e61145fd/pydub-0.25.1.tar.gz", hash = "sha256:980a33ce9949cab2a569606b65674d748ecbca4f0796887fd6f46173a7b0d30f", size = 38326 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a6/53/d78dc063216e62fc55f6b2eebb447f6a4b0a59f55c8406376f76bf959b08/pydub-0.25.1-py2.py3-none-any.whl", hash = "sha256:65617e33033874b59d87db603aa1ed450633288aefead953b30bded59cb599a6", size = 32327 }, -] - [[package]] name = "pygments" -version = "2.19.0" +version = "2.19.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d3/c0/9c9832e5be227c40e1ce774d493065f83a91d6430baa7e372094e9683a45/pygments-2.19.0.tar.gz", hash = "sha256:afc4146269910d4bdfabcd27c24923137a74d562a23a320a41a55ad303e19783", size = 4967733 } +sdist = { url = "https://files.pythonhosted.org/packages/7c/2d/c3338d48ea6cc0feb8446d8e6937e1408088a72a39937982cc6111d17f84/pygments-2.19.1.tar.gz", hash = "sha256:61c16d2a8576dc0649d9f39e089b5f02bcd27fba10d8fb4dcc28173f7a45151f", size = 4968581 } wheels = [ - { url = "https://files.pythonhosted.org/packages/20/dc/fde3e7ac4d279a331676829af4afafd113b34272393d73f610e8f0329221/pygments-2.19.0-py3-none-any.whl", hash = "sha256:4755e6e64d22161d5b61432c0600c923c5927214e7c956e31c23923c89251a9b", size = 1225305 }, + { url = "https://files.pythonhosted.org/packages/8a/0b/9fcc47d19c48b59121088dd6da2488a49d5f72dacf8262e2790a1d2c7d15/pygments-2.19.1-py3-none-any.whl", hash = "sha256:9ea1544ad55cecf4b8242fab6dd35a93bbce657034b0611ee383099054ab6d8c", size = 1225293 }, ] [[package]] @@ -3914,20 +3914,20 @@ sdist = { url = "https://files.pythonhosted.org/packages/ce/af/409edba35fc597f1e [[package]] name = "pymilvus" -version = "2.5.3" +version = "2.4.9" source = { registry = "https://pypi.org/simple" } dependencies = [ + { name = "environs", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "milvus-lite", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "python-dotenv", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "setuptools", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "ujson", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a9/8a/a10d29f5d9c9c33ac71db4594e3e6230279d557d6bd5fde6f99d1edfc360/pymilvus-2.5.3.tar.gz", hash = "sha256:68bc3797b7a14c494caf116cee888894ffd6eba7b96a3ac841be85d60694cc5d", size = 1258217 } +sdist = { url = "https://files.pythonhosted.org/packages/1c/e4/208ac8d384bdcfa1a2983a6394705edccfd15a99f6f0e478ea0400fc1c73/pymilvus-2.4.9.tar.gz", hash = "sha256:0937663700007c23a84cfc0656160b301f6ff9247aaec4c96d599a6b43572136", size = 1219775 } wheels = [ - { url = "https://files.pythonhosted.org/packages/7e/ef/2a5682e02ef69465f7a50aa48fd9ac3fe12a3f653f51cbdc211a28557efc/pymilvus-2.5.3-py3-none-any.whl", hash = "sha256:64ca63594284586937274800be27a402f3be2d078130bf81d94ab8d7798ac9c8", size = 229867 }, + { url = "https://files.pythonhosted.org/packages/0e/98/0d79ebcc04e8a469f796e644302edee4368927a268f11afc298b6bd76e1f/pymilvus-2.4.9-py3-none-any.whl", hash = "sha256:45313607d2c164064bdc44e0f933cb6d6afa92e9efcc7f357c5240c57db58fbe", size = 201144 }, ] [[package]] @@ -4029,14 +4029,14 @@ wheels = [ [[package]] name = "pytest-asyncio" -version = "0.25.1" +version = "0.25.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "pytest", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/4b/04/0477a4bdd176ad678d148c075f43620b3f7a060ff61c7da48500b1fa8a75/pytest_asyncio-0.25.1.tar.gz", hash = "sha256:79be8a72384b0c917677e00daa711e07db15259f4d23203c59012bcd989d4aee", size = 53760 } +sdist = { url = "https://files.pythonhosted.org/packages/72/df/adcc0d60f1053d74717d21d58c0048479e9cab51464ce0d2965b086bd0e2/pytest_asyncio-0.25.2.tar.gz", hash = "sha256:3f8ef9a98f45948ea91a0ed3dc4268b5326c0e7bce73892acc654df4262ad45f", size = 53950 } wheels = [ - { url = "https://files.pythonhosted.org/packages/81/fb/efc7226b384befd98d0e00d8c4390ad57f33c8fde00094b85c5e07897def/pytest_asyncio-0.25.1-py3-none-any.whl", hash = "sha256:c84878849ec63ff2ca509423616e071ef9cd8cc93c053aa33b5b8fb70a990671", size = 19357 }, + { url = "https://files.pythonhosted.org/packages/61/d8/defa05ae50dcd6019a95527200d3b3980043df5aa445d40cb0ef9f7f98ab/pytest_asyncio-0.25.2-py3-none-any.whl", hash = "sha256:0d0bb693f7b99da304a0634afc0a4b19e49d5e0de2d670f38dc4bfa5727c5075", size = 19400 }, ] [[package]] @@ -4630,24 +4630,24 @@ wheels = [ [[package]] name = "safetensors" -version = "0.5.0" +version = "0.5.2" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5d/b3/1d9000e9d0470499d124ca63c6908f8092b528b48bd95ba11507e14d9dba/safetensors-0.5.0.tar.gz", hash = "sha256:c47b34c549fa1e0c655c4644da31332c61332c732c47c8dd9399347e9aac69d1", size = 65660 } +sdist = { url = "https://files.pythonhosted.org/packages/f4/4f/2ef9ef1766f8c194b01b67a63a444d2e557c8fe1d82faf3ebd85f370a917/safetensors-0.5.2.tar.gz", hash = "sha256:cb4a8d98ba12fa016f4241932b1fc5e702e5143f5374bba0bbcf7ddc1c4cf2b8", size = 66957 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0f/ee/0fd61b99bc58db736a3ab3d97d49d4a11afe71ee0aad85b25d6c4235b743/safetensors-0.5.0-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:c683b9b485bee43422ba2855f72777c37647190281e03da4c8d2a69fa5336558", size = 426509 }, - { url = "https://files.pythonhosted.org/packages/51/aa/de1a11aa056d0241f95d5de9dbb1ac2dabaf3df5c568f9375451fd593c95/safetensors-0.5.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6106aa835deb7263f7014f74c05842ab828d6c11d789f2e7e98f26b1a305e72d", size = 408471 }, - { url = "https://files.pythonhosted.org/packages/a5/c7/84b821bd90547a909053a8526ff70446f062287cda20d0ec024c1a1f80f6/safetensors-0.5.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a1349611f74f55c5ee1c1c144c536a2743c38f7d8bf60b9fc8267e0efc0591a2", size = 449638 }, - { url = "https://files.pythonhosted.org/packages/b5/25/3d20bb9f669fec704e01d70849e9c6c054601efe9b5e784ce9a865cf3c52/safetensors-0.5.0-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:56d936028ac799e18644b08a91fd98b4b62ae3dcd0440b1cfcb56535785589f1", size = 458246 }, - { url = "https://files.pythonhosted.org/packages/31/35/68e1c39c4ad6a2f9373fc89588c0fbd29b1899c57c3a6482fc8e42fa4c8f/safetensors-0.5.0-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a2f26afada2233576ffea6b80042c2c0a8105c164254af56168ec14299ad3122", size = 509573 }, - { url = "https://files.pythonhosted.org/packages/85/b0/79927c6d4f70232f04a46785ea8b0ed0f70f9be74d17e0a90e1890523553/safetensors-0.5.0-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:20067e7a5e63f0cbc88457b2a1161e70ff73af4cc3a24bce90309430cd6f6e7e", size = 525555 }, - { url = "https://files.pythonhosted.org/packages/a6/83/ca8c1af662a20a545c174b8949e63865b747c180b607260eed83c1d38c72/safetensors-0.5.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:649d6a4aa34d5174ae87289068ccc2fec2a1a998ecf83425aa5a42c3eff69bcf", size = 461294 }, - { url = "https://files.pythonhosted.org/packages/81/ef/1d11d08b14b36e3e3d701629c9685ad95c3afee7da2851658d6c65cad9be/safetensors-0.5.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:debff88f41d569a3e93a955469f83864e432af35bb34b16f65a9ddf378daa3ae", size = 490593 }, - { url = "https://files.pythonhosted.org/packages/f6/9a/50bf824a26d768d33485b7208ba5e6a173a80a2633be5e213a2494d1569b/safetensors-0.5.0-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:bdf6a3e366ea8ba1a0538db6099229e95811194432c684ea28ea7ae28763b8dc", size = 628142 }, - { url = "https://files.pythonhosted.org/packages/28/22/dc5ae22523b8221017dbf6984fedfe2c6f35ff4cc76e80bbab2b9e14cc8a/safetensors-0.5.0-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:0371afd84c200a80eb7103bf715108b0c3846132fb82453ae018609a15551580", size = 721377 }, - { url = "https://files.pythonhosted.org/packages/fe/87/36323e8058e7101ef0101fde6d71c375a9ab6059d3d9501fe8fb8d13a45a/safetensors-0.5.0-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:5ec7fc8c3d2f32ebf1c7011bc886b362e53ee0a1ec6d828c39d531fed8b325d6", size = 659192 }, - { url = "https://files.pythonhosted.org/packages/dd/2f/8d526f06bb192b45b4e0fec94284d568497e6e19620c834373749a5f9787/safetensors-0.5.0-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:53715e4ea0ef23c08f004baae0f609a7773de7d4148727760417c6760cfd6b76", size = 632231 }, - { url = "https://files.pythonhosted.org/packages/d3/68/1166bba02f77c811d17766e54a54d7714c1276f54bfcf60d50bb9326a1b4/safetensors-0.5.0-cp38-abi3-win32.whl", hash = "sha256:b85565bc2f0456961a788d2f11d9d892eec46603db0e4923aa9512c2355aa727", size = 290608 }, - { url = "https://files.pythonhosted.org/packages/0c/ab/a428973e43a77791d2fd4b6425f4fd82e9f8559b32222c861acbbd7bc910/safetensors-0.5.0-cp38-abi3-win_amd64.whl", hash = "sha256:f451941f8aa11e7be5c3fa450e264609a2b1e65fa38ae590a74e55a94d646b76", size = 303322 }, + { url = "https://files.pythonhosted.org/packages/96/d1/017e31e75e274492a11a456a9e7c171f8f7911fe50735b4ec6ff37221220/safetensors-0.5.2-cp38-abi3-macosx_10_12_x86_64.whl", hash = "sha256:45b6092997ceb8aa3801693781a71a99909ab9cc776fbc3fa9322d29b1d3bef2", size = 427067 }, + { url = "https://files.pythonhosted.org/packages/24/84/e9d3ff57ae50dd0028f301c9ee064e5087fe8b00e55696677a0413c377a7/safetensors-0.5.2-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:6d0d6a8ee2215a440e1296b843edf44fd377b055ba350eaba74655a2fe2c4bae", size = 408856 }, + { url = "https://files.pythonhosted.org/packages/f1/1d/fe95f5dd73db16757b11915e8a5106337663182d0381811c81993e0014a9/safetensors-0.5.2-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:86016d40bcaa3bcc9a56cd74d97e654b5f4f4abe42b038c71e4f00a089c4526c", size = 450088 }, + { url = "https://files.pythonhosted.org/packages/cf/21/e527961b12d5ab528c6e47b92d5f57f33563c28a972750b238b871924e49/safetensors-0.5.2-cp38-abi3-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:990833f70a5f9c7d3fc82c94507f03179930ff7d00941c287f73b6fcbf67f19e", size = 458966 }, + { url = "https://files.pythonhosted.org/packages/a5/8b/1a037d7a57f86837c0b41905040369aea7d8ca1ec4b2a77592372b2ec380/safetensors-0.5.2-cp38-abi3-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dfa7c2f3fe55db34eba90c29df94bcdac4821043fc391cb5d082d9922013869", size = 509915 }, + { url = "https://files.pythonhosted.org/packages/61/3d/03dd5cfd33839df0ee3f4581a20bd09c40246d169c0e4518f20b21d5f077/safetensors-0.5.2-cp38-abi3-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:46ff2116150ae70a4e9c490d2ab6b6e1b1b93f25e520e540abe1b81b48560c3a", size = 527664 }, + { url = "https://files.pythonhosted.org/packages/c5/dc/8952caafa9a10a3c0f40fa86bacf3190ae7f55fa5eef87415b97b29cb97f/safetensors-0.5.2-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3ab696dfdc060caffb61dbe4066b86419107a24c804a4e373ba59be699ebd8d5", size = 461978 }, + { url = "https://files.pythonhosted.org/packages/60/da/82de1fcf1194e3dbefd4faa92dc98b33c06bed5d67890e0962dd98e18287/safetensors-0.5.2-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.whl", hash = "sha256:03c937100f38c9ff4c1507abea9928a6a9b02c9c1c9c3609ed4fb2bf413d4975", size = 491253 }, + { url = "https://files.pythonhosted.org/packages/5a/9a/d90e273c25f90c3ba1b0196a972003786f04c39e302fbd6649325b1272bb/safetensors-0.5.2-cp38-abi3-musllinux_1_2_aarch64.whl", hash = "sha256:a00e737948791b94dad83cf0eafc09a02c4d8c2171a239e8c8572fe04e25960e", size = 628644 }, + { url = "https://files.pythonhosted.org/packages/70/3c/acb23e05aa34b4f5edd2e7f393f8e6480fbccd10601ab42cd03a57d4ab5f/safetensors-0.5.2-cp38-abi3-musllinux_1_2_armv7l.whl", hash = "sha256:d3a06fae62418ec8e5c635b61a8086032c9e281f16c63c3af46a6efbab33156f", size = 721648 }, + { url = "https://files.pythonhosted.org/packages/71/45/eaa3dba5253a7c6931230dc961641455710ab231f8a89cb3c4c2af70f8c8/safetensors-0.5.2-cp38-abi3-musllinux_1_2_i686.whl", hash = "sha256:1506e4c2eda1431099cebe9abf6c76853e95d0b7a95addceaa74c6019c65d8cf", size = 659588 }, + { url = "https://files.pythonhosted.org/packages/b0/71/2f9851164f821064d43b481ddbea0149c2d676c4f4e077b178e7eeaa6660/safetensors-0.5.2-cp38-abi3-musllinux_1_2_x86_64.whl", hash = "sha256:5c5b5d9da594f638a259fca766046f44c97244cc7ab8bef161b3e80d04becc76", size = 632533 }, + { url = "https://files.pythonhosted.org/packages/00/f1/5680e2ef61d9c61454fad82c344f0e40b8741a9dbd1e31484f0d31a9b1c3/safetensors-0.5.2-cp38-abi3-win32.whl", hash = "sha256:fe55c039d97090d1f85277d402954dd6ad27f63034fa81985a9cc59655ac3ee2", size = 291167 }, + { url = "https://files.pythonhosted.org/packages/86/ca/aa489392ec6fb59223ffce825461e1f811a3affd417121a2088be7a5758b/safetensors-0.5.2-cp38-abi3-win_amd64.whl", hash = "sha256:78abdddd03a406646107f973c7843276e7b64e5e32623529dc17f3d94a20f589", size = 303756 }, ] [[package]] @@ -4812,9 +4812,6 @@ onnx = [ ] openai-realtime = [ { name = "openai", extra = ["realtime"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pyaudio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "pydub", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "sounddevice", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] pandas = [ { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -4899,14 +4896,12 @@ requires-dist = [ { name = "pybars4", specifier = "~=0.9" }, { name = "pydantic", specifier = ">=2.0,!=2.10.0,!=2.10.1,!=2.10.2,!=2.10.3,<2.11" }, { name = "pydantic-settings", specifier = "~=2.0" }, - { name = "pydub", marker = "extra == 'openai-realtime'" }, { name = "pymilvus", marker = "extra == 'milvus'", specifier = ">=2.3,<2.6" }, { name = "pymongo", marker = "extra == 'mongo'", specifier = ">=4.8.0,<4.11" }, { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = "~=1.9" }, { name = "redis", extras = ["hiredis"], marker = "extra == 'redis'", specifier = "~=5.0" }, { name = "redisvl", marker = "extra == 'redis'", specifier = ">=0.3.6" }, { name = "sentence-transformers", marker = "extra == 'hugging-face'", specifier = ">=2.2,<4.0" }, - { name = "sounddevice", marker = "extra == 'openai-realtime'" }, { name = "torch", marker = "extra == 'hugging-face'", specifier = "==2.5.1" }, { name = "transformers", extras = ["torch"], marker = "extra == 'hugging-face'", specifier = "~=4.28" }, { name = "types-redis", marker = "extra == 'redis'", specifier = "~=4.6.0.20240425" }, @@ -4950,11 +4945,11 @@ wheels = [ [[package]] name = "setuptools" -version = "75.7.0" +version = "75.8.0" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ac/57/e6f0bde5a2c333a32fbcce201f906c1fd0b3a7144138712a5e9d9598c5ec/setuptools-75.7.0.tar.gz", hash = "sha256:886ff7b16cd342f1d1defc16fc98c9ce3fde69e087a4e1983d7ab634e5f41f4f", size = 1338616 } +sdist = { url = "https://files.pythonhosted.org/packages/92/ec/089608b791d210aec4e7f97488e67ab0d33add3efccb83a056cbafe3a2a6/setuptools-75.8.0.tar.gz", hash = "sha256:c5afc8f407c626b8313a86e10311dd3f661c6cd9c09d4bf8c15c0e11f9f2b0e6", size = 1343222 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4e/6e/abdfaaf5c294c553e7a81cf5d801fbb4f53f5c5b6646de651f92a2667547/setuptools-75.7.0-py3-none-any.whl", hash = "sha256:84fb203f278ebcf5cd08f97d3fb96d3fbed4b629d500b29ad60d11e00769b183", size = 1224467 }, + { url = "https://files.pythonhosted.org/packages/69/8a/b9dc7678803429e4a3bc9ba462fa3dd9066824d3c607490235c6a796be5a/setuptools-75.8.0-py3-none-any.whl", hash = "sha256:e3982f444617239225d675215d51f6ba05f845d4eec313da4418fdbb56fb27e3", size = 1228782 }, ] [[package]] @@ -5110,21 +5105,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/93/84a16940c44f6ec62cf334f25aed3128a514dffc361397eee09421a1c7f2/snoop-0.6.0-py3-none-any.whl", hash = "sha256:f5ea9060e65594bf404e6841086b4a964cc27bc30569109c91a470f948b0f729", size = 27461 }, ] -[[package]] -name = "sounddevice" -version = "0.5.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cffi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/80/2d/b04ae180312b81dbb694504bee170eada5372242e186f6298139fd3a0513/sounddevice-0.5.1.tar.gz", hash = "sha256:09ca991daeda8ce4be9ac91e15a9a81c8f81efa6b695a348c9171ea0c16cb041", size = 52896 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/06/d1/464b5fca3decdd0cfec8c47f7b4161a0b12972453201c1bf03811f367c5e/sounddevice-0.5.1-py3-none-any.whl", hash = "sha256:e2017f182888c3f3c280d9fbac92e5dbddac024a7e3442f6e6116bd79dab8a9c", size = 32276 }, - { url = "https://files.pythonhosted.org/packages/6f/f6/6703fe7cf3d7b7279040c792aeec6334e7305956aba4a80f23e62c8fdc44/sounddevice-0.5.1-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:d16cb23d92322526a86a9490c427bf8d49e273d9ccc0bd096feecd229cde6031", size = 107916 }, - { url = "https://files.pythonhosted.org/packages/57/a5/78a5e71f5ec0faedc54f4053775d61407bfbd7d0c18228c7f3d4252fd276/sounddevice-0.5.1-py3-none-win32.whl", hash = "sha256:d84cc6231526e7a08e89beff229c37f762baefe5e0cc2747cbe8e3a565470055", size = 312494 }, - { url = "https://files.pythonhosted.org/packages/af/9b/15217b04f3b36d30de55fef542389d722de63f1ad81f9c72d8afc98cb6ab/sounddevice-0.5.1-py3-none-win_amd64.whl", hash = "sha256:4313b63f2076552b23ac3e0abd3bcfc0c1c6a696fc356759a13bd113c9df90f1", size = 363634 }, -] - [[package]] name = "soupsieve" version = "2.6" @@ -5358,7 +5338,7 @@ wheels = [ [[package]] name = "transformers" -version = "4.48.0" +version = "4.47.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "filelock", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -5372,9 +5352,9 @@ dependencies = [ { name = "tokenizers", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ea/71/93a6331682d6f15adf7d646956db0c43e5f1759bbbd05f2ef53029bae107/transformers-4.48.0.tar.gz", hash = "sha256:03fdfcbfb8b0367fb6c9fbe9d1c9aa54dfd847618be9b52400b2811d22799cb1", size = 8372101 } +sdist = { url = "https://files.pythonhosted.org/packages/15/1a/936aeb4f88112f670b604f5748034568dbc2b9bbb457a8d4518b1a15510a/transformers-4.47.1.tar.gz", hash = "sha256:6c29c05a5f595e278481166539202bf8641281536df1c42357ee58a45d0a564a", size = 8707421 } wheels = [ - { url = "https://files.pythonhosted.org/packages/45/d6/a69764e89fc5c2c957aa473881527c8c35521108d553df703e9ba703daeb/transformers-4.48.0-py3-none-any.whl", hash = "sha256:6d3de6d71cb5f2a10f9775ccc17abce9620195caaf32ec96542bd2a6937f25b0", size = 9673380 }, + { url = "https://files.pythonhosted.org/packages/f2/3a/8bdab26e09c5a242182b7ba9152e216d5ab4ae2d78c4298eb4872549cd35/transformers-4.47.1-py3-none-any.whl", hash = "sha256:d2f5d19bb6283cd66c893ec7e6d931d6370bbf1cc93633326ff1f41a40046c9c", size = 10133598 }, ] [package.optional-dependencies] From 9778a5c9dd2e7e7f1d2fe7ebd79c8237c6b7497f Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 9 Jan 2025 16:47:55 +0100 Subject: [PATCH 03/25] updated note --- .../semantic_kernel/connectors/ai/realtime_client_base.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index c5d092d50870..ebdd4eed3739 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -17,22 +17,22 @@ #### # TODO (eavanvalkenburg): Move to ADR # Receiving: -# Option 1: Events and Contents split (current) +# Option 1: Events and Contents split # - content received through main receive_content method # - events received through event callback handlers # Option 2: Everything is Content # - content (events as new Content Type) received through main receive_content method -# Option 3: Everything is Event +# Option 3: Everything is Event (current) # - receive_content method is removed # - events received through main listen method # - default event handlers added for things like errors and function calling # - built-in vs custom event handling - separate or not? # Sending: -# Option 1: Events and Contents split (current) +# Option 1: Events and Contents split # - send_content and send_event # Option 2: Everything is Content # - single method needed, with EventContent type support -# Option 3: Everything is Event +# Option 3: Everything is Event (current) # - send_event method only, Content is part of event data #### From 7cb5683a36270e73dba6ac84c6f8adaec61d8844 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 9 Jan 2025 17:01:06 +0100 Subject: [PATCH 04/25] reverted some changes --- .../connectors/ai/chat_completion_client_base.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py index c2fd0877ca09..eaf372b1f858 100644 --- a/python/semantic_kernel/connectors/ai/chat_completion_client_base.py +++ b/python/semantic_kernel/connectors/ai/chat_completion_client_base.py @@ -221,7 +221,7 @@ async def get_streaming_chat_message_contents( if not self.SUPPORTS_FUNCTION_CALLING: async for streaming_chat_message_contents in self._inner_get_streaming_chat_message_contents( - chat_history, settings, **kwargs + chat_history, settings ): yield streaming_chat_message_contents return @@ -246,7 +246,7 @@ async def get_streaming_chat_message_contents( or not settings.function_choice_behavior.auto_invoke_kernel_functions ): async for streaming_chat_message_contents in self._inner_get_streaming_chat_message_contents( - chat_history, settings, **kwargs + chat_history, settings ): yield streaming_chat_message_contents return @@ -258,7 +258,7 @@ async def get_streaming_chat_message_contents( all_messages: list["StreamingChatMessageContent"] = [] function_call_returned = False async for messages in self._inner_get_streaming_chat_message_contents( - chat_history, settings, request_index, **kwargs + chat_history, settings, request_index ): for msg in messages: if msg is not None: @@ -309,7 +309,6 @@ async def get_streaming_chat_message_contents( function_invoke_attempt=request_index, ) if self._yield_function_result_messages(function_result_messages): - await self._streaming_function_call_result_callback(function_result_messages) yield function_result_messages if any(result.terminate for result in results if result is not None): From f924f4e087aeabfb52cb1a15efe40a01c3441688 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 10 Jan 2025 13:52:55 +0100 Subject: [PATCH 05/25] WIP ADR --- docs/decisions/00XX-realtime-api-clients.md | 158 +++++++++ python/pyproject.toml | 3 +- .../open_ai_realtime_execution_settings.py | 8 +- .../open_ai/services/open_ai_realtime_base.py | 4 +- python/uv.lock | 323 +++++++++--------- 5 files changed, 328 insertions(+), 168 deletions(-) create mode 100644 docs/decisions/00XX-realtime-api-clients.md diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md new file mode 100644 index 000000000000..81d9d6fdf4e7 --- /dev/null +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -0,0 +1,158 @@ +--- +# These are optional elements. Feel free to remove any of them. +status: {proposed } +contact: {Eduard van Valkenburg} +date: {2025-01-10} +deciders: { Eduard van Valkenburg, Mark Wallace, Ben Thomas, Roger Barreto} +consulted: +informed: +--- + +# Realtime API Clients + +## Context and Problem Statement + +Multiple model providers are starting to enable realtime voice-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. The key addition that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. + +The way these API's work at this time is through either websockets or WebRTC. In both cases there are events being sent to and from the service, some events contain content, text, audio, or video (so far only sending, not receiving), while some events are "control" events, like content created, function call requested, etc. Sending events include, sending content, either voice, text or function call output, or events, like committing the input audio and requesting a response. + +Both the OpenAI and Google realtime api's are in preview/beta, this means there might be breaking changes in the way they work coming in the future, therefore the clients built to support these API's are going to be experimental until the API's stabilize. + +One feature that we need to consider if and how to deal with is whether or not a service uses Voice Activated Detection, OpenAI supports turning that off and allows parameters for how it behaves, while Google has it on by default and it cannot be configured. + +### Event types + +Client side events: +| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | +|-------------------| ------------------------------------|-------------------------|------------------------| + | Control | Configure session | `session.update` | `BidiGenerateContentSetup` | + | Content | Send voice input | `input_audio_buffer.append` | `BidiGenerateContentRealtimeInput` | + | Control | Commit input and request response | `input_audio_buffer.commit` | `-` | + | Control | Clean audio input buffer | `input_audio_buffer.clear` | `-` | + | Content | Send text input | `conversation.item.create` | `BidiGenerateContentClientContent` | + | Control | Interrupt audio | `conversation.item.truncate` | `-`| + | Control | Delete content | `conversation.item.delete` | `-`| +| Control | Respond to function call request | `conversation.item.create` | `BidiGenerateContentToolResponse`| +| Control | Ask for response | `response.create` | `-`| +| Control | Cancel response | `response.cancel` | `-`| + +Server side events: +| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | +|----------------------------|-------------------------------------|-------------------------|------------------------| +| Control | Error | `error` | `-` | +| Control | Session created | `session.created` | `BidiGenerateContentSetupComplete` | +| Control | Session updated | `session.updated` | `BidiGenerateContentSetupComplete` | +| Control | Conversation created | `conversation.created` | `-` | +| Control | Input audio buffer committed | `input_audio_buffer.committed` | `-` | +| Control | Input audio buffer cleared | `input_audio_buffer.cleared` | `-` | +| Control | Input audio buffer speech started | `input_audio_buffer.speech_started` | `-` | +| Control | Input audio buffer speech stopped | `input_audio_buffer.speech_stopped` | `-` | +| Content | Conversation item created | `conversation.item.created` | `-` | +| Content | Input audio transcription completed | `conversation.item.input_audio_transcription.completed` | +| Content | Input audio transcription failed | `conversation.item.input_audio_transcription.failed` | +| Control | Conversation item truncated | `conversation.item.truncated` | `-` | +| Control | Conversation item deleted | `conversation.item.deleted` | `-` | +| Control | Response created | `response.created` | `-` | +| Control | Response done | `response.done` | `-` | +| Content | Response output item added | `response.output_item.added` | `-` | +| Content | Response output item done | `response.output_item.done` | `-` | +| Content | Response content part added | `response.content_part.added` | `-` | +| Content | Response content part done | `response.content_part.done` | `-` | +| Content | Response text delta | `response.text.delta` | `BidiGenerateContentServerContent` | +| Content | Response text done | `response.text.done` | `-` | +| Content | Response audio transcript delta | `response.audio_transcript.delta` | `BidiGenerateContentServerContent` | +| Content | Response audio transcript done | `response.audio_transcript.done` | `-` | +| Content | Response audio delta | `response.audio.delta` | `BidiGenerateContentServerContent` | +| Content | Response audio done | `response.audio.done` | `-` | +| Content | Response function call arguments delta | `response.function_call_arguments.delta` | `BidiGenerateContentToolCall` | +| Content | Response function call arguments done | `response.function_call_arguments.done` | `-` | +| Control | Function call cancelled | `-` | `BidiGenerateContentToolCallCancellation` | +| Control | Rate limits updated | `rate_limits.updated` | `-` | + + + + +## Decision Drivers + +- Simple programming model that is likely able to handle future realtime api's and evolution of the existing ones. +- Support for the most common scenario's and content, extensible for the rest. +- Natively integrated with Semantic Kernel especially for content types and function calling. + +- … + +## Considered Options + +Both the sending and receiving side of these integrations need to decide how to deal with the api's. + +- Treat content events separate from control events +- Treat everything as content items +- Treat everything as events + +### Treat content events separate from control events +This would mean there are two mechanisms in the clients, one deals with content, and one with control events. + +- Pro: + - strongly typed responses for known content + - easy to use as the main interactions are clear with familiar SK content types, the rest goes through a separate mechanism +- Con: + - new content support requires updates in the codebase and can be considered breaking (potentitally sending additional types back) + - additional complexity in dealing with two streams of data + +### Treat everything as content items + + +## Decision Outcome + +Chosen option: "{title of option 1}", because +{justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force {force} | … | comes out best (see below)}. + + + +### Consequences + +- Good, because {positive consequence, e.g., improvement of one or more desired qualities, …} +- Bad, because {negative consequence, e.g., compromising one or more desired qualities, …} +- … + + + +## Validation + +{describe how the implementation of/compliance with the ADR is validated. E.g., by a review or an ArchUnit test} + + + +## Pros and Cons of the Options + +### {title of option 1} + + + +{example | description | pointer to more information | …} + +- Good, because {argument a} +- Good, because {argument b} + +- Neutral, because {argument c} +- Bad, because {argument d} +- … + +### {title of other option} + +{example | description | pointer to more information | …} + +- Good, because {argument a} +- Good, because {argument b} +- Neutral, because {argument c} +- Bad, because {argument d} +- … + + + +## More Information + +{You might want to provide additional evidence/confidence for the decision outcome here and/or +document the team agreement on the decision and/or +define when this decision when and how the decision should be realized and if/when it should be re-visited and/or +how the decision is validated. +Links to other decisions and resources might appear here as well.} diff --git a/python/pyproject.toml b/python/pyproject.toml index d828b1ec5aa9..b5efc0723cf3 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -61,7 +61,8 @@ chroma = [ ] google = [ "google-cloud-aiplatform ~= 1.60", - "google-generativeai ~= 0.7" + "google-generativeai ~= 0.7", + "google-genai ~= 0.4" ] hugging_face = [ "transformers[torch] ~= 4.28", diff --git a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py index 480e2ed1373f..a26237b78b84 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py +++ b/python/semantic_kernel/connectors/ai/open_ai/prompt_execution_settings/open_ai_realtime_execution_settings.py @@ -9,6 +9,12 @@ from semantic_kernel.kernel_pydantic import KernelBaseModel +class InputAudioTranscription(KernelBaseModel): + """Input audio transcription settings.""" + + model: Literal["whisper-1"] | None = None + + class TurnDetection(KernelBaseModel): """Turn detection settings.""" @@ -28,7 +34,7 @@ class OpenAIRealtimeExecutionSettings(PromptExecutionSettings): voice: str | None = None input_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | None = None output_audio_format: Literal["pcm16", "g711_ulaw", "g711_alaw"] | None = None - input_audio_transcription: dict[str, Any] | None = None + input_audio_transcription: InputAudioTranscription | None = None turn_detection: TurnDetection | None = None tools: Annotated[ list[dict[str, Any]] | None, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py index 4175d9449b2e..64b647f44ee8 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -100,7 +100,7 @@ class ListenEvents(str, Enum): CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated" CONVERSATION_ITEM_DELETED = "conversation.item.deleted" RESPONSE_CREATED = "response.created" - RESPONSE_DONE = "response.done" + RESPONSE_DONE = "response.done" # contains usage info -> log RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added" RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done" RESPONSE_CONTENT_PART_ADDED = "response.content_part.added" @@ -421,6 +421,8 @@ async def response_function_call_arguments_done_callback( chat_history = ChatHistory() await kernel.invoke_function_call(item, chat_history) await self.send_event(SendEvents.CONVERSATION_ITEM_CREATE, item=chat_history.messages[-1]) + # The model doesn't start responding to the tool call automatically, so triggering it here. + await self.send_event(SendEvents.RESPONSE_CREATE) return chat_history.messages[-1], False def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: diff --git a/python/uv.lock b/python/uv.lock index b53948ef0d64..161b91c05ab9 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -1,18 +1,18 @@ version = 1 requires-python = ">=3.10" resolution-markers = [ - "python_full_version >= '3.13' and sys_platform == 'darwin'", - "python_full_version == '3.12.*' and sys_platform == 'darwin'", - "python_full_version == '3.11.*' and sys_platform == 'darwin'", "python_full_version < '3.11' and sys_platform == 'darwin'", - "python_full_version >= '3.13' and sys_platform == 'linux'", - "python_full_version == '3.12.*' and sys_platform == 'linux'", - "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.11.*' and sys_platform == 'darwin'", + "python_full_version == '3.12.*' and sys_platform == 'darwin'", + "python_full_version >= '3.13' and sys_platform == 'darwin'", "python_full_version < '3.11' and sys_platform == 'linux'", - "python_full_version >= '3.13' and sys_platform == 'win32'", - "python_full_version == '3.12.*' and sys_platform == 'win32'", - "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'linux'", + "python_full_version == '3.12.*' and sys_platform == 'linux'", + "python_full_version >= '3.13' and sys_platform == 'linux'", "python_full_version < '3.11' and sys_platform == 'win32'", + "python_full_version == '3.11.*' and sys_platform == 'win32'", + "python_full_version == '3.12.*' and sys_platform == 'win32'", + "python_full_version >= '3.13' and sys_platform == 'win32'", ] supported-markers = [ "sys_platform == 'darwin'", @@ -414,30 +414,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.35.95" +version = "1.35.96" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "s3transfer", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/97/b5/b961eb4d803ade4c90113b254630482f59a5d89b84e6939c9d4c7893d0c7/boto3-1.35.95.tar.gz", hash = "sha256:d5671226819f6a78e31b1f37bd60f194afb8203254a543d06bdfb76de7d79236", size = 111014 } +sdist = { url = "https://files.pythonhosted.org/packages/fc/6c/ea481adf32791472885664224ac8e5269a4429db2e510d9fa56c493407e9/boto3-1.35.96.tar.gz", hash = "sha256:bace02ef2181d176cedc1f8f90c95c301bb7c555db124cf80bc193cbb52a7c64", size = 110999 } wheels = [ - { url = "https://files.pythonhosted.org/packages/3a/e1/1910792d5eceff426bd9048c454766df720cb0fd26473907fbfd1c64d518/boto3-1.35.95-py3-none-any.whl", hash = "sha256:c81223488607457dacb7829ee0c99803c664593b34a2b0f86c71d421e7c3469a", size = 139182 }, + { url = "https://files.pythonhosted.org/packages/17/07/a1da47e567f7550783a6def2b1840d1b69c1f0cd4933e6f1c5942ff4a6c6/boto3-1.35.96-py3-none-any.whl", hash = "sha256:e6acb2380791b13d8fd55062d9bbc6e27c3ddb3e73cff71c4ca02e6743780c67", size = 139181 }, ] [[package]] name = "botocore" -version = "1.35.95" +version = "1.35.96" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "python-dateutil", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "urllib3", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/d8/b7/1cf5da213ce2e00a5bcd480a9355aa23f787e11ef63eecb637bd7e48deef/botocore-1.35.95.tar.gz", hash = "sha256:b03d2d7cc58a16aa96a7e8f21941b766e98abc6ea74f61a63de7dc26c03641d3", size = 13489115 } +sdist = { url = "https://files.pythonhosted.org/packages/d3/b2/9b2558e3f0094eb4829338bca777fc0747ad69fa8fe0b5f692d7e4e86bea/botocore-1.35.96.tar.gz", hash = "sha256:385fd406ed14bdd624e082d3e15dd6575d490d5d7374fb02f0a798c3ca9ea802", size = 13488154 } wheels = [ - { url = "https://files.pythonhosted.org/packages/cf/97/e001bbab0773b66a5512022cc26deb82b8743f16ba5662fe762019c4c52c/botocore-1.35.95-py3-none-any.whl", hash = "sha256:a672406f748ad6a5b2fb7ea0d8394539eb4fda5332fc5373467d232c4bb27b12", size = 13289333 }, + { url = "https://files.pythonhosted.org/packages/65/bc/9ba93a90b3f53afdd5d27c4a0b7bc19b5b9d6ad0e1489b4c5cd47ef6fbe4/botocore-1.35.96-py3-none-any.whl", hash = "sha256:b5f4cf11372aeccf87bb0b6148a020212c4c42fb5bcdebb6590bb10f6612b98e", size = 13289712 }, ] [[package]] @@ -646,7 +646,7 @@ wheels = [ [[package]] name = "chromadb" -version = "0.5.20" +version = "0.6.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "bcrypt", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -678,9 +678,9 @@ dependencies = [ { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "uvicorn", extra = ["standard"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/03/31/6c8e05405bb02b4a1f71f9aa3eef242415565dabf6afc1bde7f64f726963/chromadb-0.5.20.tar.gz", hash = "sha256:19513a23b2d20059866216bfd80195d1d4a160ffba234b8899f5e80978160ca7", size = 33664540 } +sdist = { url = "https://files.pythonhosted.org/packages/d1/c5/d2b4219fdee424e881608da681c3c63b73d68dc6667bd2df14a4d9bb308d/chromadb-0.6.2.tar.gz", hash = "sha256:e9e11f04d3850796711ee05dad4e918c75ec7b62ab9cbe7b4588b68a26aaea06", size = 19979649 } wheels = [ - { url = "https://files.pythonhosted.org/packages/5f/7a/10bf5dc92d13cc03230190fcc5016a0b138d99e5b36b8b89ee0fe1680e10/chromadb-0.5.20-py3-none-any.whl", hash = "sha256:9550ba1b6dce911e35cac2568b301badf4b42f457b99a432bdeec2b6b9dd3680", size = 617884 }, + { url = "https://files.pythonhosted.org/packages/bb/1c/2b77093f4191ad2d1ab70b9215cb6bc9f43350aa3e9e54a44304c8379335/chromadb-0.6.2-py3-none-any.whl", hash = "sha256:77a5e07097e36cdd49d8d2925d0c4d28291cabc9677787423d2cc7c426e8895b", size = 606162 }, ] [[package]] @@ -984,19 +984,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4c/a3/ac312faeceffd2d8f86bc6dcb5c401188ba5a01bc88e69bed97578a0dfcd/durationpy-0.9-py3-none-any.whl", hash = "sha256:e65359a7af5cedad07fb77a2dd3f390f8eb0b74cb845589fa6c057086834dd38", size = 3461 }, ] -[[package]] -name = "environs" -version = "9.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "marshmallow", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "python-dotenv", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d4/e3/c3c6c76f3dbe3e019e9a451b35bf9f44690026a5bb1232f7b77097b72ff5/environs-9.5.0.tar.gz", hash = "sha256:a76307b36fbe856bdca7ee9161e6c466fd7fcffc297109a118c59b54e27e30c9", size = 20795 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ca/5e/f0f217dc393372681bfe05c50f06a212e78d0a3fee907a74ab451ec1dcdb/environs-9.5.0-py2.py3-none-any.whl", hash = "sha256:1e549569a3de49c05f856f40bce86979e7d5ffbbc4398e7f338574c220189124", size = 12548 }, -] - [[package]] name = "eval-type-backport" version = "0.2.2" @@ -1220,7 +1207,7 @@ grpc = [ [[package]] name = "google-api-python-client" -version = "2.157.0" +version = "2.158.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-api-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -1229,9 +1216,9 @@ dependencies = [ { name = "httplib2", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "uritemplate", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/43/ec/f9f61460adf4e16bfe64c59a8e708e2209521cd48d6ad6d8b1e14e7627f1/google_api_python_client-2.157.0.tar.gz", hash = "sha256:2ee342d0967ad1cedec43ccd7699671d94bff151e1f06833ea81303f9a6d86fd", size = 12275652 } +sdist = { url = "https://files.pythonhosted.org/packages/b3/d7/ed1626cb92ffe68a17c5e5b3f0331e20f3ff6ef24deedffd4a70db49e0b0/google_api_python_client-2.158.0.tar.gz", hash = "sha256:b6664597a9955e04977a62752e33fe44cb35c580e190c1cb08a041893172bd67", size = 12277176 } wheels = [ - { url = "https://files.pythonhosted.org/packages/16/33/be58f58b63ffcc6b57e52428b388dbc94fb008baae60e81b205ea64e5baa/google_api_python_client-2.157.0-py2.py3-none-any.whl", hash = "sha256:0b0231db106324c659bf8b85f390391c00da57a60ebc4271e33def7aac198c75", size = 12787473 }, + { url = "https://files.pythonhosted.org/packages/b0/91/02f0f4938957892224a1fd8a9c031175a28036d4c8ee538972922a342efd/google_api_python_client-2.158.0-py2.py3-none-any.whl", hash = "sha256:36f8c8d2e79e50f76790ca5946d2f3f8333e210dc8539a6c88e0742416474ad2", size = 12789578 }, ] [[package]] @@ -1374,6 +1361,22 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/14/fb/54deefe679b7d1c1cc81d83396fcf28ad1a66d213bddeb275a8d28665918/google_crc32c-1.6.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:18e311c64008f1f1379158158bb3f0c8d72635b9eb4f9545f8cf990c5668e59d", size = 27866 }, ] +[[package]] +name = "google-genai" +version = "0.4.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "google-auth", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pillow", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "requests", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "websockets", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/8f/fa/e8c81d37ffe7d8aa05573494735cdc432a97b77f641a08caa959de19523d/google_genai-0.4.0.tar.gz", hash = "sha256:d14ce2e941063092cfc98726aeabcae44f179456e3a4906ee5f28dc91b0663fb", size = 107625 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9d/ac/cf91960fc842f8c3387be8abeaa01deb0e6b20a72a028b70107f58e13150/google_genai-0.4.0-py3-none-any.whl", hash = "sha256:2cbfea3cb47d4ac54ee3d3f9ecd79ff72298cac13e150828afdc5ed62768ed00", size = 113562 }, +] + [[package]] name = "google-generativeai" version = "0.8.3" @@ -1437,122 +1440,122 @@ wheels = [ [[package]] name = "grpcio" -version = "1.69.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/e4/87/06a145284cbe86c91ca517fe6b57be5efbb733c0d6374b407f0992054d18/grpcio-1.69.0.tar.gz", hash = "sha256:936fa44241b5379c5afc344e1260d467bee495747eaf478de825bab2791da6f5", size = 12738244 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b0/6e/2f8ee5fb65aef962d0bd7e46b815e7b52820687e29c138eaee207a688abc/grpcio-1.69.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:2060ca95a8db295ae828d0fc1c7f38fb26ccd5edf9aa51a0f44251f5da332e97", size = 5190753 }, - { url = "https://files.pythonhosted.org/packages/89/07/028dcda44d40f9488f0a0de79c5ffc80e2c1bc5ed89da9483932e3ea67cf/grpcio-1.69.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:2e52e107261fd8fa8fa457fe44bfadb904ae869d87c1280bf60f93ecd3e79278", size = 11096752 }, - { url = "https://files.pythonhosted.org/packages/99/a0/c727041b1410605ba38b585b6b52c1a289d7fcd70a41bccbc2c58fc643b2/grpcio-1.69.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:316463c0832d5fcdb5e35ff2826d9aa3f26758d29cdfb59a368c1d6c39615a11", size = 5705442 }, - { url = "https://files.pythonhosted.org/packages/7a/2f/1c53f5d127ff882443b19c757d087da1908f41c58c4b098e8eaf6b2bb70a/grpcio-1.69.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:26c9a9c4ac917efab4704b18eed9082ed3b6ad19595f047e8173b5182fec0d5e", size = 6333796 }, - { url = "https://files.pythonhosted.org/packages/cc/f6/2017da2a1b64e896af710253e5bfbb4188605cdc18bce3930dae5cdbf502/grpcio-1.69.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90b3646ced2eae3a0599658eeccc5ba7f303bf51b82514c50715bdd2b109e5ec", size = 5954245 }, - { url = "https://files.pythonhosted.org/packages/c1/65/1395bec928e99ba600464fb01b541e7e4cdd462e6db25259d755ef9f8d02/grpcio-1.69.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3b75aea7c6cb91b341c85e7c1d9db1e09e1dd630b0717f836be94971e015031e", size = 6664854 }, - { url = "https://files.pythonhosted.org/packages/40/57/8b3389cfeb92056c8b44288c9c4ed1d331bcad0215c4eea9ae4629e156d9/grpcio-1.69.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:5cfd14175f9db33d4b74d63de87c64bb0ee29ce475ce3c00c01ad2a3dc2a9e51", size = 6226854 }, - { url = "https://files.pythonhosted.org/packages/cc/61/1f2bbeb7c15544dffc98b3f65c093e746019995e6f1e21dc3655eec3dc23/grpcio-1.69.0-cp310-cp310-win32.whl", hash = "sha256:9031069d36cb949205293cf0e243abd5e64d6c93e01b078c37921493a41b72dc", size = 3662734 }, - { url = "https://files.pythonhosted.org/packages/ef/ba/bf1a6d9f5c17d2da849793d72039776c56c98c889c9527f6721b6ee57e6e/grpcio-1.69.0-cp310-cp310-win_amd64.whl", hash = "sha256:cc89b6c29f3dccbe12d7a3b3f1b3999db4882ae076c1c1f6df231d55dbd767a5", size = 4410306 }, - { url = "https://files.pythonhosted.org/packages/8d/cd/ca256aeef64047881586331347cd5a68a4574ba1a236e293cd8eba34e355/grpcio-1.69.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:8de1b192c29b8ce45ee26a700044717bcbbd21c697fa1124d440548964328561", size = 5198734 }, - { url = "https://files.pythonhosted.org/packages/37/3f/10c1e5e0150bf59aa08ea6aebf38f87622f95f7f33f98954b43d1b2a3200/grpcio-1.69.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:7e76accf38808f5c5c752b0ab3fd919eb14ff8fafb8db520ad1cc12afff74de6", size = 11135285 }, - { url = "https://files.pythonhosted.org/packages/08/61/61cd116a572203a740684fcba3fef37a3524f1cf032b6568e1e639e59db0/grpcio-1.69.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:d5658c3c2660417d82db51e168b277e0ff036d0b0f859fa7576c0ffd2aec1442", size = 5699468 }, - { url = "https://files.pythonhosted.org/packages/01/f1/a841662e8e2465ba171c973b77d18fa7438ced535519b3c53617b7e6e25c/grpcio-1.69.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5494d0e52bf77a2f7eb17c6da662886ca0a731e56c1c85b93505bece8dc6cf4c", size = 6332337 }, - { url = "https://files.pythonhosted.org/packages/62/b1/c30e932e02c2e0bfdb8df46fe3b0c47f518fb04158ebdc0eb96cc97d642f/grpcio-1.69.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4ed866f9edb574fd9be71bf64c954ce1b88fc93b2a4cbf94af221e9426eb14d6", size = 5949844 }, - { url = "https://files.pythonhosted.org/packages/5e/cb/55327d43b6286100ffae7d1791be6178d13c917382f3e9f43f82e8b393cf/grpcio-1.69.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c5ba38aeac7a2fe353615c6b4213d1fbb3a3c34f86b4aaa8be08baaaee8cc56d", size = 6661828 }, - { url = "https://files.pythonhosted.org/packages/6f/e4/120d72ae982d51cb9cabcd9672f8a1c6d62011b493a4d049d2abdf564db0/grpcio-1.69.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:f79e05f5bbf551c4057c227d1b041ace0e78462ac8128e2ad39ec58a382536d2", size = 6226026 }, - { url = "https://files.pythonhosted.org/packages/96/e8/2cc15f11db506d7b1778f0587fa7bdd781602b05b3c4d75b7ca13de33d62/grpcio-1.69.0-cp311-cp311-win32.whl", hash = "sha256:bf1f8be0da3fcdb2c1e9f374f3c2d043d606d69f425cd685110dd6d0d2d61258", size = 3662653 }, - { url = "https://files.pythonhosted.org/packages/42/78/3c5216829a48237fcb71a077f891328a435e980d9757a9ebc49114d88768/grpcio-1.69.0-cp311-cp311-win_amd64.whl", hash = "sha256:fb9302afc3a0e4ba0b225cd651ef8e478bf0070cf11a529175caecd5ea2474e7", size = 4412824 }, - { url = "https://files.pythonhosted.org/packages/61/1d/8f28f147d7f3f5d6b6082f14e1e0f40d58e50bc2bd30d2377c730c57a286/grpcio-1.69.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:fc18a4de8c33491ad6f70022af5c460b39611e39578a4d84de0fe92f12d5d47b", size = 5161414 }, - { url = "https://files.pythonhosted.org/packages/35/4b/9ab8ea65e515e1844feced1ef9e7a5d8359c48d986c93f3d2a2006fbdb63/grpcio-1.69.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:0f0270bd9ffbff6961fe1da487bdcd594407ad390cc7960e738725d4807b18c4", size = 11108909 }, - { url = "https://files.pythonhosted.org/packages/99/68/1856fde2b3c3162bdfb9845978608deef3606e6907fdc2c87443fce6ecd0/grpcio-1.69.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:dc48f99cc05e0698e689b51a05933253c69a8c8559a47f605cff83801b03af0e", size = 5658302 }, - { url = "https://files.pythonhosted.org/packages/3e/21/3fa78d38dc5080d0d677103fad3a8cd55091635cc2069a7c06c7a54e6c4d/grpcio-1.69.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1e925954b18d41aeb5ae250262116d0970893b38232689c4240024e4333ac084", size = 6306201 }, - { url = "https://files.pythonhosted.org/packages/f3/cb/5c47b82fd1baf43dba973ae399095d51aaf0085ab0439838b4cbb1e87e3c/grpcio-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:87d222569273720366f68a99cb62e6194681eb763ee1d3b1005840678d4884f9", size = 5919649 }, - { url = "https://files.pythonhosted.org/packages/c6/67/59d1a56a0f9508a29ea03e1ce800bdfacc1f32b4f6b15274b2e057bf8758/grpcio-1.69.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:b62b0f41e6e01a3e5082000b612064c87c93a49b05f7602fe1b7aa9fd5171a1d", size = 6648974 }, - { url = "https://files.pythonhosted.org/packages/f8/fe/ca70c14d98c6400095f19a0f4df8273d09c2106189751b564b26019f1dbe/grpcio-1.69.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db6f9fd2578dbe37db4b2994c94a1d9c93552ed77dca80e1657bb8a05b898b55", size = 6215144 }, - { url = "https://files.pythonhosted.org/packages/b3/94/b2b0a9fd487fc8262e20e6dd0ec90d9fa462c82a43b4855285620f6e9d01/grpcio-1.69.0-cp312-cp312-win32.whl", hash = "sha256:b192b81076073ed46f4b4dd612b8897d9a1e39d4eabd822e5da7b38497ed77e1", size = 3644552 }, - { url = "https://files.pythonhosted.org/packages/93/99/81aec9f85412e3255a591ae2ccb799238e074be774e5f741abae08a23418/grpcio-1.69.0-cp312-cp312-win_amd64.whl", hash = "sha256:1227ff7836f7b3a4ab04e5754f1d001fa52a730685d3dc894ed8bc262cc96c01", size = 4399532 }, - { url = "https://files.pythonhosted.org/packages/54/47/3ff4501365f56b7cc16617695dbd4fd838c5e362bc7fa9fee09d592f7d78/grpcio-1.69.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:a78a06911d4081a24a1761d16215a08e9b6d4d29cdbb7e427e6c7e17b06bcc5d", size = 5162928 }, - { url = "https://files.pythonhosted.org/packages/c0/63/437174c5fa951052c9ecc5f373f62af6f3baf25f3f5ef35cbf561806b371/grpcio-1.69.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:dc5a351927d605b2721cbb46158e431dd49ce66ffbacb03e709dc07a491dde35", size = 11103027 }, - { url = "https://files.pythonhosted.org/packages/53/df/53566a6fdc26b6d1f0585896e1cc4825961039bca5a6a314ff29d79b5d5b/grpcio-1.69.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:3629d8a8185f5139869a6a17865d03113a260e311e78fbe313f1a71603617589", size = 5659277 }, - { url = "https://files.pythonhosted.org/packages/e6/4c/b8a0c4f71498b6f9be5ca6d290d576cf2af9d95fd9827c47364f023969ad/grpcio-1.69.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9a281878feeb9ae26db0622a19add03922a028d4db684658f16d546601a4870", size = 6305255 }, - { url = "https://files.pythonhosted.org/packages/ef/55/d9aa05eb3dfcf6aa946aaf986740ec07fc5189f20e2cbeb8c5d278ffd00f/grpcio-1.69.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8cc614e895177ab7e4b70f154d1a7c97e152577ea101d76026d132b7aaba003b", size = 5920240 }, - { url = "https://files.pythonhosted.org/packages/ea/eb/774b27c51e3e386dfe6c491a710f6f87ffdb20d88ec6c3581e047d9354a2/grpcio-1.69.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:1ee76cd7e2e49cf9264f6812d8c9ac1b85dda0eaea063af07292400f9191750e", size = 6652974 }, - { url = "https://files.pythonhosted.org/packages/59/98/96de14e6e7d89123813d58c246d9b0f1fbd24f9277f5295264e60861d9d6/grpcio-1.69.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:0470fa911c503af59ec8bc4c82b371ee4303ececbbdc055f55ce48e38b20fd67", size = 6215757 }, - { url = "https://files.pythonhosted.org/packages/7d/5b/ce922e0785910b10756fabc51fd294260384a44bea41651dadc4e47ddc82/grpcio-1.69.0-cp313-cp313-win32.whl", hash = "sha256:b650f34aceac8b2d08a4c8d7dc3e8a593f4d9e26d86751ebf74ebf5107d927de", size = 3642488 }, - { url = "https://files.pythonhosted.org/packages/5d/04/11329e6ca1ceeb276df2d9c316b5e170835a687a4d0f778dba8294657e36/grpcio-1.69.0-cp313-cp313-win_amd64.whl", hash = "sha256:028337786f11fecb5d7b7fa660475a06aabf7e5e52b5ac2df47414878c0ce7ea", size = 4399968 }, +version = "1.67.1" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/20/53/d9282a66a5db45981499190b77790570617a604a38f3d103d0400974aeb5/grpcio-1.67.1.tar.gz", hash = "sha256:3dc2ed4cabea4dc14d5e708c2b426205956077cc5de419b4d4079315017e9732", size = 12580022 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/4e/cd/f6ca5c49aa0ae7bc6d0757f7dae6f789569e9490a635eaabe02bc02de7dc/grpcio-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:8b0341d66a57f8a3119b77ab32207072be60c9bf79760fa609c5609f2deb1f3f", size = 5112450 }, + { url = "https://files.pythonhosted.org/packages/d4/f0/d9bbb4a83cbee22f738ee7a74aa41e09ccfb2dcea2cc30ebe8dab5b21771/grpcio-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:f5a27dddefe0e2357d3e617b9079b4bfdc91341a91565111a21ed6ebbc51b22d", size = 10937518 }, + { url = "https://files.pythonhosted.org/packages/5b/17/0c5dbae3af548eb76669887642b5f24b232b021afe77eb42e22bc8951d9c/grpcio-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:43112046864317498a33bdc4797ae6a268c36345a910de9b9c17159d8346602f", size = 5633610 }, + { url = "https://files.pythonhosted.org/packages/17/48/e000614e00153d7b2760dcd9526b95d72f5cfe473b988e78f0ff3b472f6c/grpcio-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:c9b929f13677b10f63124c1a410994a401cdd85214ad83ab67cc077fc7e480f0", size = 6240678 }, + { url = "https://files.pythonhosted.org/packages/64/19/a16762a70eeb8ddfe43283ce434d1499c1c409ceec0c646f783883084478/grpcio-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e7d1797a8a3845437d327145959a2c0c47c05947c9eef5ff1a4c80e499dcc6fa", size = 5884528 }, + { url = "https://files.pythonhosted.org/packages/6b/dc/bd016aa3684914acd2c0c7fa4953b2a11583c2b844f3d7bae91fa9b98fbb/grpcio-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:0489063974d1452436139501bf6b180f63d4977223ee87488fe36858c5725292", size = 6583680 }, + { url = "https://files.pythonhosted.org/packages/1a/93/1441cb14c874f11aa798a816d582f9da82194b6677f0f134ea53d2d5dbeb/grpcio-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:9fd042de4a82e3e7aca44008ee2fb5da01b3e5adb316348c21980f7f58adc311", size = 6162967 }, + { url = "https://files.pythonhosted.org/packages/29/e9/9295090380fb4339b7e935b9d005fa9936dd573a22d147c9e5bb2df1b8d4/grpcio-1.67.1-cp310-cp310-win32.whl", hash = "sha256:638354e698fd0c6c76b04540a850bf1db27b4d2515a19fcd5cf645c48d3eb1ed", size = 3616336 }, + { url = "https://files.pythonhosted.org/packages/ce/de/7c783b8cb8f02c667ca075c49680c4aeb8b054bc69784bcb3e7c1bbf4985/grpcio-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:608d87d1bdabf9e2868b12338cd38a79969eaf920c89d698ead08f48de9c0f9e", size = 4352071 }, + { url = "https://files.pythonhosted.org/packages/59/2c/b60d6ea1f63a20a8d09c6db95c4f9a16497913fb3048ce0990ed81aeeca0/grpcio-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:7818c0454027ae3384235a65210bbf5464bd715450e30a3d40385453a85a70cb", size = 5119075 }, + { url = "https://files.pythonhosted.org/packages/b3/9a/e1956f7ca582a22dd1f17b9e26fcb8229051b0ce6d33b47227824772feec/grpcio-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:ea33986b70f83844cd00814cee4451055cd8cab36f00ac64a31f5bb09b31919e", size = 11009159 }, + { url = "https://files.pythonhosted.org/packages/43/a8/35fbbba580c4adb1d40d12e244cf9f7c74a379073c0a0ca9d1b5338675a1/grpcio-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:c7a01337407dd89005527623a4a72c5c8e2894d22bead0895306b23c6695698f", size = 5629476 }, + { url = "https://files.pythonhosted.org/packages/77/c9/864d336e167263d14dfccb4dbfa7fce634d45775609895287189a03f1fc3/grpcio-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:80b866f73224b0634f4312a4674c1be21b2b4afa73cb20953cbbb73a6b36c3cc", size = 6239901 }, + { url = "https://files.pythonhosted.org/packages/f7/1e/0011408ebabf9bd69f4f87cc1515cbfe2094e5a32316f8714a75fd8ddfcb/grpcio-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f9fff78ba10d4250bfc07a01bd6254a6d87dc67f9627adece85c0b2ed754fa96", size = 5881010 }, + { url = "https://files.pythonhosted.org/packages/b4/7d/fbca85ee9123fb296d4eff8df566f458d738186d0067dec6f0aa2fd79d71/grpcio-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:8a23cbcc5bb11ea7dc6163078be36c065db68d915c24f5faa4f872c573bb400f", size = 6580706 }, + { url = "https://files.pythonhosted.org/packages/75/7a/766149dcfa2dfa81835bf7df623944c1f636a15fcb9b6138ebe29baf0bc6/grpcio-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1a65b503d008f066e994f34f456e0647e5ceb34cfcec5ad180b1b44020ad4970", size = 6161799 }, + { url = "https://files.pythonhosted.org/packages/09/13/5b75ae88810aaea19e846f5380611837de411181df51fd7a7d10cb178dcb/grpcio-1.67.1-cp311-cp311-win32.whl", hash = "sha256:e29ca27bec8e163dca0c98084040edec3bc49afd10f18b412f483cc68c712744", size = 3616330 }, + { url = "https://files.pythonhosted.org/packages/aa/39/38117259613f68f072778c9638a61579c0cfa5678c2558706b10dd1d11d3/grpcio-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:786a5b18544622bfb1e25cc08402bd44ea83edfb04b93798d85dca4d1a0b5be5", size = 4354535 }, + { url = "https://files.pythonhosted.org/packages/6e/25/6f95bd18d5f506364379eabc0d5874873cc7dbdaf0757df8d1e82bc07a88/grpcio-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:267d1745894200e4c604958da5f856da6293f063327cb049a51fe67348e4f953", size = 5089809 }, + { url = "https://files.pythonhosted.org/packages/10/3f/d79e32e5d0354be33a12db2267c66d3cfeff700dd5ccdd09fd44a3ff4fb6/grpcio-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:85f69fdc1d28ce7cff8de3f9c67db2b0ca9ba4449644488c1e0303c146135ddb", size = 10981985 }, + { url = "https://files.pythonhosted.org/packages/21/f2/36fbc14b3542e3a1c20fb98bd60c4732c55a44e374a4eb68f91f28f14aab/grpcio-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:f26b0b547eb8d00e195274cdfc63ce64c8fc2d3e2d00b12bf468ece41a0423a0", size = 5588770 }, + { url = "https://files.pythonhosted.org/packages/0d/af/bbc1305df60c4e65de8c12820a942b5e37f9cf684ef5e49a63fbb1476a73/grpcio-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4422581cdc628f77302270ff839a44f4c24fdc57887dc2a45b7e53d8fc2376af", size = 6214476 }, + { url = "https://files.pythonhosted.org/packages/92/cf/1d4c3e93efa93223e06a5c83ac27e32935f998bc368e276ef858b8883154/grpcio-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1d7616d2ded471231c701489190379e0c311ee0a6c756f3c03e6a62b95a7146e", size = 5850129 }, + { url = "https://files.pythonhosted.org/packages/ae/ca/26195b66cb253ac4d5ef59846e354d335c9581dba891624011da0e95d67b/grpcio-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8a00efecde9d6fcc3ab00c13f816313c040a28450e5e25739c24f432fc6d3c75", size = 6568489 }, + { url = "https://files.pythonhosted.org/packages/d1/94/16550ad6b3f13b96f0856ee5dfc2554efac28539ee84a51d7b14526da985/grpcio-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:699e964923b70f3101393710793289e42845791ea07565654ada0969522d0a38", size = 6149369 }, + { url = "https://files.pythonhosted.org/packages/33/0d/4c3b2587e8ad7f121b597329e6c2620374fccbc2e4e1aa3c73ccc670fde4/grpcio-1.67.1-cp312-cp312-win32.whl", hash = "sha256:4e7b904484a634a0fff132958dabdb10d63e0927398273917da3ee103e8d1f78", size = 3599176 }, + { url = "https://files.pythonhosted.org/packages/7d/36/0c03e2d80db69e2472cf81c6123aa7d14741de7cf790117291a703ae6ae1/grpcio-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:5721e66a594a6c4204458004852719b38f3d5522082be9061d6510b455c90afc", size = 4346574 }, + { url = "https://files.pythonhosted.org/packages/12/d2/2f032b7a153c7723ea3dea08bffa4bcaca9e0e5bdf643ce565b76da87461/grpcio-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:aa0162e56fd10a5547fac8774c4899fc3e18c1aa4a4759d0ce2cd00d3696ea6b", size = 5091487 }, + { url = "https://files.pythonhosted.org/packages/d0/ae/ea2ff6bd2475a082eb97db1104a903cf5fc57c88c87c10b3c3f41a184fc0/grpcio-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:beee96c8c0b1a75d556fe57b92b58b4347c77a65781ee2ac749d550f2a365dc1", size = 10943530 }, + { url = "https://files.pythonhosted.org/packages/07/62/646be83d1a78edf8d69b56647327c9afc223e3140a744c59b25fbb279c3b/grpcio-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:a93deda571a1bf94ec1f6fcda2872dad3ae538700d94dc283c672a3b508ba3af", size = 5589079 }, + { url = "https://files.pythonhosted.org/packages/d0/25/71513d0a1b2072ce80d7f5909a93596b7ed10348b2ea4fdcbad23f6017bf/grpcio-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0e6f255980afef598a9e64a24efce87b625e3e3c80a45162d111a461a9f92955", size = 6213542 }, + { url = "https://files.pythonhosted.org/packages/76/9a/d21236297111052dcb5dc85cd77dc7bf25ba67a0f55ae028b2af19a704bc/grpcio-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9e838cad2176ebd5d4a8bb03955138d6589ce9e2ce5d51c3ada34396dbd2dba8", size = 5850211 }, + { url = "https://files.pythonhosted.org/packages/2d/fe/70b1da9037f5055be14f359026c238821b9bcf6ca38a8d760f59a589aacd/grpcio-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:a6703916c43b1d468d0756c8077b12017a9fcb6a1ef13faf49e67d20d7ebda62", size = 6572129 }, + { url = "https://files.pythonhosted.org/packages/74/0d/7df509a2cd2a54814598caf2fb759f3e0b93764431ff410f2175a6efb9e4/grpcio-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:917e8d8994eed1d86b907ba2a61b9f0aef27a2155bca6cbb322430fc7135b7bb", size = 6149819 }, + { url = "https://files.pythonhosted.org/packages/0a/08/bc3b0155600898fd10f16b79054e1cca6cb644fa3c250c0fe59385df5e6f/grpcio-1.67.1-cp313-cp313-win32.whl", hash = "sha256:e279330bef1744040db8fc432becc8a727b84f456ab62b744d3fdb83f327e121", size = 3596561 }, + { url = "https://files.pythonhosted.org/packages/5a/96/44759eca966720d0f3e1b105c43f8ad4590c97bf8eb3cd489656e9590baa/grpcio-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:fa0c739ad8b1996bd24823950e3cb5152ae91fca1c09cc791190bf1627ffefba", size = 4346042 }, ] [[package]] name = "grpcio-health-checking" -version = "1.69.0" +version = "1.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/ef/b8/d6d485e27d60174ba22c25587c1a97512c6a800633cfd6a8cd7943ad66e0/grpcio_health_checking-1.69.0.tar.gz", hash = "sha256:ff6e1d38c2a300b1bbd296916fbd9165667bc4b5a8557f99dd4226d4f9e8f4c1", size = 16809 } +sdist = { url = "https://files.pythonhosted.org/packages/64/dd/e3b339fa44dc75b501a1a22cb88f1af5b1f8c964488f19c4de4cfbbf05ba/grpcio_health_checking-1.67.1.tar.gz", hash = "sha256:ca90fa76a6afbb4fda71d734cb9767819bba14928b91e308cffbb0c311eb941e", size = 16775 } wheels = [ - { url = "https://files.pythonhosted.org/packages/a4/07/8d68bb1821dc46dfb5b702374c5d06e9c0013afb08fa92516ebd8f963ef3/grpcio_health_checking-1.69.0-py3-none-any.whl", hash = "sha256:d2d0eec7e3af245863fd4997e2942d27c0868fbd61ffa4d14bc492c3e2c67127", size = 18923 }, + { url = "https://files.pythonhosted.org/packages/5c/8d/7a9878dca6616b48093d71c52d0bc79cb2dd1a2698ff6f5ce7406306de12/grpcio_health_checking-1.67.1-py3-none-any.whl", hash = "sha256:93753da5062152660aef2286c9b261e07dd87124a65e4dc9fbd47d1ce966b39d", size = 18924 }, ] [[package]] name = "grpcio-status" -version = "1.69.0" +version = "1.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "googleapis-common-protos", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/02/35/52dc0d8300f879dbf9cdc95764cee9f56d5a212998cfa1a8871b262df2a4/grpcio_status-1.69.0.tar.gz", hash = "sha256:595ef84e5178d6281caa732ccf68ff83259241608d26b0e9c40a5e66eee2a2d2", size = 13662 } +sdist = { url = "https://files.pythonhosted.org/packages/be/c7/fe0e79a80ac6346e0c6c0a24e9e3cbc3ae1c2a009acffb59eab484a6f69b/grpcio_status-1.67.1.tar.gz", hash = "sha256:2bf38395e028ceeecfd8866b081f61628114b384da7d51ae064ddc8d766a5d11", size = 13673 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f6/e2/346a766a4232f74f45f8bc70e636fc3a6677e6bc3893382187829085f12e/grpcio_status-1.69.0-py3-none-any.whl", hash = "sha256:d6b2a3c9562c03a817c628d7ba9a925e209c228762d6d7677ae5c9401a542853", size = 14428 }, + { url = "https://files.pythonhosted.org/packages/05/18/56999a1da3577d8ccc8698a575d6638e15fe25650cc88b2ce0a087f180b9/grpcio_status-1.67.1-py3-none-any.whl", hash = "sha256:16e6c085950bdacac97c779e6a502ea671232385e6e37f258884d6883392c2bd", size = 14427 }, ] [[package]] name = "grpcio-tools" -version = "1.69.0" +version = "1.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "setuptools", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/64/ec/1c25136ca1697eaa09a02effe3e74959fd9fb6aba9960d7340dd6341c5ce/grpcio_tools-1.69.0.tar.gz", hash = "sha256:3e1a98f4d9decb84979e1ddd3deb09c0a33a84b6e3c0776d5bde4097e3ab66dd", size = 5323319 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/00/90/7df7326552fec627adcf3880cf13e9a5b23c090bbcedba367f64fa2bb54b/grpcio_tools-1.69.0-cp310-cp310-linux_armv7l.whl", hash = "sha256:8c210630faa581c3bd08953dac4ad21a7f49862f3b92d69686e9b436d2f1265d", size = 2388795 }, - { url = "https://files.pythonhosted.org/packages/e2/03/6ccaa58b3ca1734d0868a389148e22ac15248a9be4c223805339f7904e31/grpcio_tools-1.69.0-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:09b66ea279fcdaebae4ec34b1baf7577af3b14322738aa980c1c33cfea71f7d7", size = 5703156 }, - { url = "https://files.pythonhosted.org/packages/c9/f6/162b456684d2444b43e45ace4e889087301e5890bbfd16ee6b2aedf36219/grpcio_tools-1.69.0-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:be94a4bfa56d356aae242cc54072c9ccc2704b659eaae2fd599a94afebf791ce", size = 2350725 }, - { url = "https://files.pythonhosted.org/packages/db/3a/2e83fea8c90b9902d68964491d014d688177a6ad0303dbbe6c2c16f25da6/grpcio_tools-1.69.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:28778debad73a8c8e0a0e07e6a2f76eecce43adbc205d17dd244d2d58bb0f0aa", size = 2727230 }, - { url = "https://files.pythonhosted.org/packages/63/06/be27b8f1811ff4cc556bdec64a9004755a929df035dc606466a75c9ac0fa/grpcio_tools-1.69.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:449308d93e4c97ae3a4503510c6d64978748ff5e21429c85da14fdc783c0f498", size = 2472752 }, - { url = "https://files.pythonhosted.org/packages/a3/43/f94578afa1535287b7b0ba39eeb23b2b8304a2a5b8e325ed7079d2ad9cba/grpcio_tools-1.69.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:b9343651e73bc6e0df6bb518c2638bf9cc2194b50d060cdbcf1b2121cd4e4ae3", size = 3344074 }, - { url = "https://files.pythonhosted.org/packages/13/d1/5f9030cbb6195f3bb182e740f349cdaa71d9c38c1b2572f401270709d7d2/grpcio_tools-1.69.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:2f08b063612553e726e328aef3a27adfaea8d92712b229012afc54d59da88a02", size = 2953778 }, - { url = "https://files.pythonhosted.org/packages/0c/cb/4812660e150d197de81296fa04ed6ad012d1aeac23bbe21be5f51493f455/grpcio_tools-1.69.0-cp310-cp310-win32.whl", hash = "sha256:599ffd39525e7bbb6412a63e56a2e6c1af8f3493fe4305260efd4a11d064cce0", size = 957556 }, - { url = "https://files.pythonhosted.org/packages/4e/c7/c7d5f5418909764e63208b9f76812db3287ece4f79500e815178194e1db9/grpcio_tools-1.69.0-cp310-cp310-win_amd64.whl", hash = "sha256:02f92e3c2bae67ece818787f8d3d89df0fa1e5e6bbb7c1493824fd5dfad886dd", size = 1114783 }, - { url = "https://files.pythonhosted.org/packages/7e/f4/575f536bada8d8f5f8943c317ae28faafe7b4aaf95ef84a599f4f3e67db3/grpcio_tools-1.69.0-cp311-cp311-linux_armv7l.whl", hash = "sha256:c18df5d1c8e163a29863583ec51237d08d7059ef8d4f7661ee6d6363d3e38fe3", size = 2388772 }, - { url = "https://files.pythonhosted.org/packages/87/94/1157342b046f51c4d076f21ef76da6d89323929b7e870389204fd49e3f09/grpcio_tools-1.69.0-cp311-cp311-macosx_10_14_universal2.whl", hash = "sha256:37876ae49235ef2e61e5059faf45dc5e7142ca54ae61aec378bb9483e0cd7e95", size = 5726348 }, - { url = "https://files.pythonhosted.org/packages/36/5c/cfd9160ef1867e025844b2695d436bb953c2d5f9c20eaaa7da6fd739ab0c/grpcio_tools-1.69.0-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:33120920e29959eaa37a1268c6a22af243d086b1a5e5222b4203e29560ece9ce", size = 2350857 }, - { url = "https://files.pythonhosted.org/packages/61/70/10614b8bc39f06548a0586fdd5d97843da4789965e758fba87726bde8c2f/grpcio_tools-1.69.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:788bb3ecd1b44664d829d319b3c1ebc15c7d7b5e7d1f22706ab57d6acd2c6301", size = 2727157 }, - { url = "https://files.pythonhosted.org/packages/37/fb/33faedb3e991dceb7a2bf802d3875bff7d6a6b6a80d314197adc73739cae/grpcio_tools-1.69.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f453b11a112e3774c8957ec2570669f3da1f7fbc8ee242482c38981496e88da2", size = 2472882 }, - { url = "https://files.pythonhosted.org/packages/41/f7/abddc158919a982f6b8e61d4a5c72569b2963304c162c3ca53c6c14d23ee/grpcio_tools-1.69.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7e5c5dc2b656755cb58b11a7e87b65258a4a8eaff01b6c30ffcb230dd447c03d", size = 3343987 }, - { url = "https://files.pythonhosted.org/packages/ba/46/e7219456aefe29137728246a67199fcbfdaa99ede93d2045a6406f0e4c0b/grpcio_tools-1.69.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:8eabf0a7a98c14322bc74f9910c96f98feebe311e085624b2d022924d4f652ca", size = 2953659 }, - { url = "https://files.pythonhosted.org/packages/74/be/262c5d2b681930f8c58012500741fe06cb40a770c9d395650efe9042467f/grpcio_tools-1.69.0-cp311-cp311-win32.whl", hash = "sha256:ad567bea43d018c2215e1db10316eda94ca19229a834a3221c15d132d24c1b8a", size = 957447 }, - { url = "https://files.pythonhosted.org/packages/8e/55/68153acca126dced35f888e708a65169df8fa8a4d5f0e78166a395e3fa9c/grpcio_tools-1.69.0-cp311-cp311-win_amd64.whl", hash = "sha256:3d64e801586dbea3530f245d48b9ed031738cc3eb099d5ce2fdb1b3dc2e1fb20", size = 1114753 }, - { url = "https://files.pythonhosted.org/packages/5b/f6/9cd1aa47556664564b873cd187d8dec978ff2f4a539d8c6d5d2f418d3d36/grpcio_tools-1.69.0-cp312-cp312-linux_armv7l.whl", hash = "sha256:8ef8efe8beac4cc1e30d41893e4096ca2601da61001897bd17441645de2d4d3c", size = 2388440 }, - { url = "https://files.pythonhosted.org/packages/62/37/0bcd8431e44b38f648f70368dd60542d10ffaffa109563349ee635013e10/grpcio_tools-1.69.0-cp312-cp312-macosx_10_14_universal2.whl", hash = "sha256:a00e87a0c5a294028115a098819899b08dd18449df5b2aac4a2b87ba865e8681", size = 5726135 }, - { url = "https://files.pythonhosted.org/packages/8b/f5/2ec994bbf522a231ce54c41a2d3621e77bece1240aafe31f12804052af0f/grpcio_tools-1.69.0-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:7722700346d5b223159532e046e51f2ff743ed4342e5fe3e0457120a4199015e", size = 2350247 }, - { url = "https://files.pythonhosted.org/packages/a9/29/9ebf54315a499a766e4c3bd53124267491162e9049c2d9ed45f43222b98f/grpcio_tools-1.69.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a934116fdf202cb675246056ee54645c743e2240632f86a37e52f91a405c7143", size = 2727994 }, - { url = "https://files.pythonhosted.org/packages/f0/2a/1a031018660b5d95c1a4c587a0babd0d28f0aa0c9a40dbca330567049a3f/grpcio_tools-1.69.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7e6a6d44359ca836acfbc58103daf94b3bb8ac919d659bb348dcd7fbecedc293", size = 2472625 }, - { url = "https://files.pythonhosted.org/packages/74/bf/76d24078e1c76976a10760c3193b6c62685a7aed64b1cb0d8242afa16f1d/grpcio_tools-1.69.0-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:e27662c0597fd1ab5399a583d358b5203edcb6fc2b29d6245099dfacd51a6ddc", size = 3344290 }, - { url = "https://files.pythonhosted.org/packages/f1/f7/4ab645e4955ca1e5240b0bbd557662cec4838f0e21e072ff40f4e191b48d/grpcio_tools-1.69.0-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:7bbb2b2fb81d95bcdd1d8331defb5f5dc256dbe423bb98b682cf129cdd432366", size = 2953592 }, - { url = "https://files.pythonhosted.org/packages/8f/32/57e67b126f209f289fc32009309d155b8dbe9ac760c32733746e4dda7b51/grpcio_tools-1.69.0-cp312-cp312-win32.whl", hash = "sha256:e11accd10cf4af5031ac86c45f1a13fb08f55e005cea070917c12e78fe6d2aa2", size = 957042 }, - { url = "https://files.pythonhosted.org/packages/19/64/7bfcb4e50a0ce87690c24696cd666f528e672119966abead09ae65a2e1da/grpcio_tools-1.69.0-cp312-cp312-win_amd64.whl", hash = "sha256:6df4c6ac109af338a8ccde29d184e0b0bdab13d78490cb360ff9b192a1aec7e2", size = 1114248 }, - { url = "https://files.pythonhosted.org/packages/0c/ef/a9867f612e3aa5e69d299e47a72ea8dafa476b1f099462c9a1223cd6a83c/grpcio_tools-1.69.0-cp313-cp313-linux_armv7l.whl", hash = "sha256:8c320c4faa1431f2e1252ef2325a970ac23b2fd04ffef6c12f96dd4552c3445c", size = 2388281 }, - { url = "https://files.pythonhosted.org/packages/4b/53/b2752d8ec338778e48d76845d605a0f8bca9e43a5f09428e5ed1a76e4e1d/grpcio_tools-1.69.0-cp313-cp313-macosx_10_14_universal2.whl", hash = "sha256:5f1224596ad74dd14444b20c37122b361c5d203b67e14e018b995f3c5d76eede", size = 5725856 }, - { url = "https://files.pythonhosted.org/packages/83/dd/195d3639634c0c1d1e48b6693c074d66a64f16c748df2f40bcee74aa04e2/grpcio_tools-1.69.0-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:965a0cf656a113bc32d15ac92ca51ed702a75d5370ae0afbdd36f818533a708a", size = 2350180 }, - { url = "https://files.pythonhosted.org/packages/8c/18/c412884fa0e888d8a271f3e31d23e3765cde0efe2404653ab67971c411c2/grpcio_tools-1.69.0-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:978835768c11a7f28778b3b7c40f839d8a57f765c315e80c4246c23900d56149", size = 2726724 }, - { url = "https://files.pythonhosted.org/packages/be/c7/dfb59b7e25d760bfdd93f0aef7dd0e2a37f8437ac3017b8b526c68764e2f/grpcio_tools-1.69.0-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:094c7cec9bd271a32dfb7c620d4a558c63fcb0122fd1651b9ed73d6afd4ae6fe", size = 2472127 }, - { url = "https://files.pythonhosted.org/packages/f2/b6/af4edf0a181fd7b148a83d491f5677d7d1c9f86f03282f8f0209d9dfb793/grpcio_tools-1.69.0-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:b51bf4981b3d7e47c2569efadff08284787124eb3dea0f63f491d39703231d3c", size = 3344015 }, - { url = "https://files.pythonhosted.org/packages/0a/9f/4c2b5ae642f7d3df73c16df6c7d53e9443cb0e49e1dcf2c8d1a49058e0b5/grpcio_tools-1.69.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:ea7aaf0dc1a828e2133357a9e9553fd1bb4e766890d52a506cc132e40632acdc", size = 2952942 }, - { url = "https://files.pythonhosted.org/packages/97/8e/6b707871db5927a17ad7475c070916bff4f32463a51552b424779236ab65/grpcio_tools-1.69.0-cp313-cp313-win32.whl", hash = "sha256:4320f11b79d3a148cc23bad1b81719ce1197808dc2406caa8a8ba0a5cfb0260d", size = 956242 }, - { url = "https://files.pythonhosted.org/packages/27/e2/b419a02b50240143605f77cd50cb07f724caf0fd35a01540a4f044ae9f21/grpcio_tools-1.69.0-cp313-cp313-win_amd64.whl", hash = "sha256:e9bae733654e0eb8ca83aa1d0d6b6c2f4a3525ce70d5ffc07df68d28f6520137", size = 1113616 }, +sdist = { url = "https://files.pythonhosted.org/packages/ae/f9/6facde12a5a8da4398a3a8947f8ba6ef33b408dfc9767c8cefc0074ddd68/grpcio_tools-1.67.1.tar.gz", hash = "sha256:d9657f5ddc62b52f58904e6054b7d8a8909ed08a1e28b734be3a707087bcf004", size = 5159073 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e5/46/668e681e2e4ca7dc80cb5ad22bc794958c8b604b5b3143f16b94be3c0118/grpcio_tools-1.67.1-cp310-cp310-linux_armv7l.whl", hash = "sha256:c701aaa51fde1f2644bd94941aa94c337adb86f25cd03cf05e37387aaea25800", size = 2308117 }, + { url = "https://files.pythonhosted.org/packages/d6/56/1c65fb7c836cd40470f1f1a88185973466241fdb42b42b7a83367c268622/grpcio_tools-1.67.1-cp310-cp310-macosx_12_0_universal2.whl", hash = "sha256:6a722bba714392de2386569c40942566b83725fa5c5450b8910e3832a5379469", size = 5500152 }, + { url = "https://files.pythonhosted.org/packages/01/ab/caf9c330241d843a83043b023e2996e959cdc2c3ab404b1a9938eb734143/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_aarch64.whl", hash = "sha256:0c7415235cb154e40b5ae90e2a172a0eb8c774b6876f53947cf0af05c983d549", size = 2282055 }, + { url = "https://files.pythonhosted.org/packages/75/e6/0cd849d140b58fedb7d3b15d907fe2eefd4dadff09b570dd687d841c5d00/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6a4c459098c4934f9470280baf9ff8b38c365e147f33c8abc26039a948a664a5", size = 2617360 }, + { url = "https://files.pythonhosted.org/packages/b9/51/bd73cd6515c2e81ba0a29b3cf6f2f62ad94737326f70b32511d1972a383e/grpcio_tools-1.67.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e89bf53a268f55c16989dab1cf0b32a5bff910762f138136ffad4146129b7a10", size = 2416028 }, + { url = "https://files.pythonhosted.org/packages/47/e5/6a16e23036f625b6d60b579996bb9bb7165485903f934d9d9d73b3f03ef5/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:f09cb3e6bcb140f57b878580cf3b848976f67faaf53d850a7da9bfac12437068", size = 3224906 }, + { url = "https://files.pythonhosted.org/packages/14/cb/230c17d4372fa46fc799a822f25fa00c8eb3f85cc86e192b9606a17f732f/grpcio_tools-1.67.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:616dd0c6686212ca90ff899bb37eb774798677e43dc6f78c6954470782d37399", size = 2870384 }, + { url = "https://files.pythonhosted.org/packages/66/fd/6d9dd3bf5982ab7d7e773f055360185e96a96cf95f2cbc7f53ded5912ef5/grpcio_tools-1.67.1-cp310-cp310-win32.whl", hash = "sha256:58a66dbb3f0fef0396737ac09d6571a7f8d96a544ce3ed04c161f3d4fa8d51cc", size = 941138 }, + { url = "https://files.pythonhosted.org/packages/6a/97/2fd5ebd996c12b2cb1e1202ee4a03cac0a65ba17d29dd34253bfe2079839/grpcio_tools-1.67.1-cp310-cp310-win_amd64.whl", hash = "sha256:89ee7c505bdf152e67c2cced6055aed4c2d4170f53a2b46a7e543d3b90e7b977", size = 1091151 }, + { url = "https://files.pythonhosted.org/packages/b5/9a/ec06547673c5001c2604637069ff8f287df1aef3f0f8809b09a1c936b049/grpcio_tools-1.67.1-cp311-cp311-linux_armv7l.whl", hash = "sha256:6d80ddd87a2fb7131d242f7d720222ef4f0f86f53ec87b0a6198c343d8e4a86e", size = 2307990 }, + { url = "https://files.pythonhosted.org/packages/ca/84/4b7c3c27a2972c00b3b6ccaadd349e0f86b7039565d3a4932e219a4d76e0/grpcio_tools-1.67.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:b655425b82df51f3bd9fd3ba1a6282d5c9ce1937709f059cb3d419b224532d89", size = 5526552 }, + { url = "https://files.pythonhosted.org/packages/a7/2d/a620e4c53a3b808ebecaa5033c2176925ee1c6cbb45c29af8bec9a249822/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_aarch64.whl", hash = "sha256:250241e6f9d20d0910a46887dfcbf2ec9108efd3b48f3fb95bb42d50d09d03f8", size = 2282137 }, + { url = "https://files.pythonhosted.org/packages/ec/29/e188b2e438781b37532abb8f10caf5b09c611a0bf9a09940b4cf303afd5b/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6008f5a5add0b6f03082edb597acf20d5a9e4e7c55ea1edac8296c19e6a0ec8d", size = 2617333 }, + { url = "https://files.pythonhosted.org/packages/86/aa/2bbccd3c34b1fa48b892fbad91525c33a8aa85cbedd50e8b0d17dc260dc3/grpcio_tools-1.67.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5eff9818c3831fa23735db1fa39aeff65e790044d0a312260a0c41ae29cc2d9e", size = 2415806 }, + { url = "https://files.pythonhosted.org/packages/db/34/99853a8ced1119937d02511476018dc1d6b295a4803d4ead5dbf9c55e9bc/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:262ab7c40113f8c3c246e28e369661ddf616a351cb34169b8ba470c9a9c3b56f", size = 3224765 }, + { url = "https://files.pythonhosted.org/packages/66/39/8537a8ace8f6242f2058677e56a429587ec731c332985af34f35d496ca58/grpcio_tools-1.67.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1eebd8c746adf5786fa4c3056258c21cc470e1eca51d3ed23a7fb6a697fe4e81", size = 2870446 }, + { url = "https://files.pythonhosted.org/packages/28/2a/5c04375adccff58647d48675e055895c31811a0ad896e4ba310833e2154d/grpcio_tools-1.67.1-cp311-cp311-win32.whl", hash = "sha256:3eff92fb8ca1dd55e3af0ef02236c648921fb7d0e8ca206b889585804b3659ae", size = 940890 }, + { url = "https://files.pythonhosted.org/packages/e6/ee/7861339c2cec8d55a5e859cf3682bda34eab5a040f95d0c80f775d6a3279/grpcio_tools-1.67.1-cp311-cp311-win_amd64.whl", hash = "sha256:1ed18281ee17e5e0f9f6ce0c6eb3825ca9b5a0866fc1db2e17fab8aca28b8d9f", size = 1091094 }, + { url = "https://files.pythonhosted.org/packages/d9/cf/7b1908ca72e484bac555431036292c48d2d6504a45e2789848cb5ff313a8/grpcio_tools-1.67.1-cp312-cp312-linux_armv7l.whl", hash = "sha256:bd5caef3a484e226d05a3f72b2d69af500dca972cf434bf6b08b150880166f0b", size = 2307645 }, + { url = "https://files.pythonhosted.org/packages/bb/15/0d1efb38af8af7e56b2342322634a3caf5f1337a6c3857a6d14aa590dfdf/grpcio_tools-1.67.1-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:48a2d63d1010e5b218e8e758ecb2a8d63c0c6016434e9f973df1c3558917020a", size = 5525468 }, + { url = "https://files.pythonhosted.org/packages/52/42/a810709099f09ade7f32990c0712c555b3d7eab6a05fb62618c17f8fe9da/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_aarch64.whl", hash = "sha256:baa64a6aa009bffe86309e236c81b02cd4a88c1ebd66f2d92e84e9b97a9ae857", size = 2281768 }, + { url = "https://files.pythonhosted.org/packages/4c/2a/64ee6cfdf1c32ef8bdd67bf04ae2f745f517f4a546281453ca1f68fa79ca/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4ab318c40b5e3c097a159035fc3e4ecfbe9b3d2c9de189e55468b2c27639a6ab", size = 2617359 }, + { url = "https://files.pythonhosted.org/packages/79/7f/1ed8cd1529253fef9cf0ef3cd8382641125a5ca2eaa08eaffbb549f84e0b/grpcio_tools-1.67.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:50eba3e31f9ac1149463ad9182a37349850904f142cffbd957cd7f54ec320b8e", size = 2415323 }, + { url = "https://files.pythonhosted.org/packages/8e/08/59f0073c58703c176c15fb1a838763b77c1c06994adba16654b92a666e1b/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:de6fbc071ecc4fe6e354a7939202191c1f1abffe37fbce9b08e7e9a5b93eba3d", size = 3225051 }, + { url = "https://files.pythonhosted.org/packages/b7/0d/a5d703214fe49d261b4b8f0a64140a4dc1f88560724a38ad937120b899ad/grpcio_tools-1.67.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:db9e87f6ea4b0ce99b2651203480585fd9e8dd0dd122a19e46836e93e3a1b749", size = 2870421 }, + { url = "https://files.pythonhosted.org/packages/ac/af/41d79cb87eae99c0348e8f1fb3dbed9e40a6f63548b216e99f4d1165fa5c/grpcio_tools-1.67.1-cp312-cp312-win32.whl", hash = "sha256:6a595a872fb720dde924c4e8200f41d5418dd6baab8cc1a3c1e540f8f4596351", size = 940542 }, + { url = "https://files.pythonhosted.org/packages/66/e5/096e12f5319835aa2bcb746d49ae62220bb48313ca649e89bdbef605c11d/grpcio_tools-1.67.1-cp312-cp312-win_amd64.whl", hash = "sha256:92eebb9b31031604ae97ea7657ae2e43149b0394af7117ad7e15894b6cc136dc", size = 1090425 }, + { url = "https://files.pythonhosted.org/packages/62/b3/91c88440c978740752d39f1abae83f21408048b98b93652ebd84f974ad3d/grpcio_tools-1.67.1-cp313-cp313-linux_armv7l.whl", hash = "sha256:9a3b9510cc87b6458b05ad49a6dee38df6af37f9ee6aa027aa086537798c3d4a", size = 2307453 }, + { url = "https://files.pythonhosted.org/packages/05/33/faf3330825463c0409fa3891bc1459bf86a00055b19790211365279538d7/grpcio_tools-1.67.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:9e4c9b9fa9b905f15d414cb7bd007ba7499f8907bdd21231ab287a86b27da81a", size = 5517975 }, + { url = "https://files.pythonhosted.org/packages/bd/78/461ab34cadbd0b5b9a0b6efedda96b58e0de471e3fa91d8e4a4e31924e1b/grpcio_tools-1.67.1-cp313-cp313-manylinux_2_17_aarch64.whl", hash = "sha256:e11a98b41af4bc88b7a738232b8fa0306ad82c79fa5d7090bb607f183a57856f", size = 2281081 }, + { url = "https://files.pythonhosted.org/packages/5f/0c/b30bdbcab1795b12e05adf30c20981c14f66198e22044edb15b3c1d9f0bc/grpcio_tools-1.67.1-cp313-cp313-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:de0fcfe61c26679d64b1710746f2891f359593f76894fcf492c37148d5694f00", size = 2616929 }, + { url = "https://files.pythonhosted.org/packages/d3/c2/a77ca68ae768f8d5f1d070ea4afc42fda40401083e7c4f5c08211e84de38/grpcio_tools-1.67.1-cp313-cp313-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae3b3e2ee5aad59dece65a613624c46a84c9582fc3642686537c6dfae8e47dc", size = 2414633 }, + { url = "https://files.pythonhosted.org/packages/39/70/8d7131dccfe4d7b739c96ada7ea9acde631f58f013eae773791fb490a3eb/grpcio_tools-1.67.1-cp313-cp313-musllinux_1_1_i686.whl", hash = "sha256:9a630f83505b6471a3094a7a372a1240de18d0cd3e64f4fbf46b361bac2be65b", size = 3224328 }, + { url = "https://files.pythonhosted.org/packages/2a/28/2d24b933ccf0d6877035aa3d5f8b64aad18c953657dd43c682b5701dc127/grpcio_tools-1.67.1-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:d85a1fcbacd3e08dc2b3d1d46b749351a9a50899fa35cf2ff040e1faf7d405ad", size = 2869640 }, + { url = "https://files.pythonhosted.org/packages/37/77/ddd2b4cc896639fb0f85fc21d5684f25080ee28845c5a4031e3dd65fdc92/grpcio_tools-1.67.1-cp313-cp313-win32.whl", hash = "sha256:778470f025f25a1fca5a48c93c0a18af395b46b12dd8df7fca63736b85181f41", size = 939997 }, + { url = "https://files.pythonhosted.org/packages/96/d0/f0855a0ccb26ffeb41e6db68b5cbb25d7e9ba1f8f19151eef36210e64efc/grpcio_tools-1.67.1-cp313-cp313-win_amd64.whl", hash = "sha256:6961da86e9856b4ddee0bf51ef6636b4bf9c29c0715aa71f3c8f027c45d42654", size = 1089819 }, ] [[package]] @@ -2206,18 +2209,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/4f/65/6079a46068dfceaeabb5dcad6d674f5f5c61a6fa5673746f42a9f4c233b3/MarkupSafe-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:e444a31f8db13eb18ada366ab3cf45fd4b31e4db1236a4448f68778c1d1a5a2f", size = 15739 }, ] -[[package]] -name = "marshmallow" -version = "3.24.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/3b/1f/52fa79445669322ee42fdd11b591c2e9c8dbab33eaf7059ca881b349ae09/marshmallow-3.24.2.tar.gz", hash = "sha256:0822c3701de396b51d3f8ac97319aea5493998ba4e7d0e4c05f6fce7777bf3a2", size = 176520 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/67/40/7802bb90b1ecbb284ae613da2cfde9ce0177b77d76cbb276acf976296aa8/marshmallow-3.24.2-py3-none-any.whl", hash = "sha256:bf3c56db473bb160e5191f1c5e32e3fc8bfb58998eb2b35d6747de023e31f9e7", size = 49333 }, -] - [[package]] name = "matplotlib-inline" version = "0.1.7" @@ -2911,7 +2902,7 @@ wheels = [ [[package]] name = "openai" -version = "1.59.5" +version = "1.59.6" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "anyio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -2923,9 +2914,9 @@ dependencies = [ { name = "tqdm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7c/b3/a99ff4f8034383147f853200ff5f6df63a8407a0061d6b3ff47914b94f4c/openai-1.59.5.tar.gz", hash = "sha256:9886e77c02dad9dc6a7b67a11ab372a56842a9b5d376aa476672175ab10e83a0", size = 344773 } +sdist = { url = "https://files.pythonhosted.org/packages/2e/7a/07fbe7bdabffd0a5be1bfe5903a02c4fff232e9acbae894014752a8e4def/openai-1.59.6.tar.gz", hash = "sha256:c7670727c2f1e4473f62fea6fa51475c8bc098c9ffb47bfb9eef5be23c747934", size = 344915 } wheels = [ - { url = "https://files.pythonhosted.org/packages/4b/a2/a64f495c016234ca4269005b19eb9193a925dcad01af95eb8fea3de4ee9c/openai-1.59.5-py3-none-any.whl", hash = "sha256:e646b44856b0dda9345d3c43639e056334d792d1690e99690313c0ef7ca4d8cc", size = 454815 }, + { url = "https://files.pythonhosted.org/packages/70/45/6de8e5fd670c804b29c777e4716f1916741c71604d5c7d952eee8432f7d3/openai-1.59.6-py3-none-any.whl", hash = "sha256:b28ed44eee3d5ebe1a3ea045ee1b4b50fea36ecd50741aaa5ce5a5559c900cb6", size = 454817 }, ] [package.optional-dependencies] @@ -3783,16 +3774,16 @@ wheels = [ [[package]] name = "pydantic" -version = "2.10.4" +version = "2.10.5" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "annotated-types", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pydantic-core", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/70/7e/fb60e6fee04d0ef8f15e4e01ff187a196fa976eb0f0ab524af4599e5754c/pydantic-2.10.4.tar.gz", hash = "sha256:82f12e9723da6de4fe2ba888b5971157b3be7ad914267dea8f05f82b28254f06", size = 762094 } +sdist = { url = "https://files.pythonhosted.org/packages/6a/c7/ca334c2ef6f2e046b1144fe4bb2a5da8a4c574e7f2ebf7e16b34a6a2fa92/pydantic-2.10.5.tar.gz", hash = "sha256:278b38dbbaec562011d659ee05f63346951b3a248a6f3642e1bc68894ea2b4ff", size = 761287 } wheels = [ - { url = "https://files.pythonhosted.org/packages/f3/26/3e1bbe954fde7ee22a6e7d31582c642aad9e84ffe4b5fb61e63b87cd326f/pydantic-2.10.4-py3-none-any.whl", hash = "sha256:597e135ea68be3a37552fb524bc7d0d66dcf93d395acd93a00682f1efcb8ee3d", size = 431765 }, + { url = "https://files.pythonhosted.org/packages/58/26/82663c79010b28eddf29dcdd0ea723439535fa917fce5905885c0e9ba562/pydantic-2.10.5-py3-none-any.whl", hash = "sha256:4dd4e322dbe55472cb7ca7e73f4b63574eecccf2835ffa2af9021ce113c83c53", size = 431426 }, ] [[package]] @@ -3914,20 +3905,20 @@ sdist = { url = "https://files.pythonhosted.org/packages/ce/af/409edba35fc597f1e [[package]] name = "pymilvus" -version = "2.4.9" +version = "2.5.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "environs", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "grpcio", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "milvus-lite", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "protobuf", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "python-dotenv", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "setuptools", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "ujson", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/1c/e4/208ac8d384bdcfa1a2983a6394705edccfd15a99f6f0e478ea0400fc1c73/pymilvus-2.4.9.tar.gz", hash = "sha256:0937663700007c23a84cfc0656160b301f6ff9247aaec4c96d599a6b43572136", size = 1219775 } +sdist = { url = "https://files.pythonhosted.org/packages/a9/8a/a10d29f5d9c9c33ac71db4594e3e6230279d557d6bd5fde6f99d1edfc360/pymilvus-2.5.3.tar.gz", hash = "sha256:68bc3797b7a14c494caf116cee888894ffd6eba7b96a3ac841be85d60694cc5d", size = 1258217 } wheels = [ - { url = "https://files.pythonhosted.org/packages/0e/98/0d79ebcc04e8a469f796e644302edee4368927a268f11afc298b6bd76e1f/pymilvus-2.4.9-py3-none-any.whl", hash = "sha256:45313607d2c164064bdc44e0f933cb6d6afa92e9efcc7f357c5240c57db58fbe", size = 201144 }, + { url = "https://files.pythonhosted.org/packages/7e/ef/2a5682e02ef69465f7a50aa48fd9ac3fe12a3f653f51cbdc211a28557efc/pymilvus-2.5.3-py3-none-any.whl", hash = "sha256:64ca63594284586937274800be27a402f3be2d078130bf81d94ab8d7798ac9c8", size = 229867 }, ] [[package]] @@ -4593,27 +4584,27 @@ wheels = [ [[package]] name = "ruff" -version = "0.8.6" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/da/00/089db7890ea3be5709e3ece6e46408d6f1e876026ec3fd081ee585fef209/ruff-0.8.6.tar.gz", hash = "sha256:dcad24b81b62650b0eb8814f576fc65cfee8674772a6e24c9b747911801eeaa5", size = 3473116 } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/28/aa07903694637c2fa394a9f4fe93cf861ad8b09f1282fa650ef07ff9fe97/ruff-0.8.6-py3-none-linux_armv6l.whl", hash = "sha256:defed167955d42c68b407e8f2e6f56ba52520e790aba4ca707a9c88619e580e3", size = 10628735 }, - { url = "https://files.pythonhosted.org/packages/2b/43/827bb1448f1fcb0fb42e9c6edf8fb067ca8244923bf0ddf12b7bf949065c/ruff-0.8.6-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:54799ca3d67ae5e0b7a7ac234baa657a9c1784b48ec954a094da7c206e0365b1", size = 10386758 }, - { url = "https://files.pythonhosted.org/packages/df/93/fc852a81c3cd315b14676db3b8327d2bb2d7508649ad60bfdb966d60738d/ruff-0.8.6-py3-none-macosx_11_0_arm64.whl", hash = "sha256:e88b8f6d901477c41559ba540beeb5a671e14cd29ebd5683903572f4b40a9807", size = 10007808 }, - { url = "https://files.pythonhosted.org/packages/94/e9/e0ed4af1794335fb280c4fac180f2bf40f6a3b859cae93a5a3ada27325ae/ruff-0.8.6-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0509e8da430228236a18a677fcdb0c1f102dd26d5520f71f79b094963322ed25", size = 10861031 }, - { url = "https://files.pythonhosted.org/packages/82/68/da0db02f5ecb2ce912c2bef2aa9fcb8915c31e9bc363969cfaaddbc4c1c2/ruff-0.8.6-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:91a7ddb221779871cf226100e677b5ea38c2d54e9e2c8ed847450ebbdf99b32d", size = 10388246 }, - { url = "https://files.pythonhosted.org/packages/ac/1d/b85383db181639019b50eb277c2ee48f9f5168f4f7c287376f2b6e2a6dc2/ruff-0.8.6-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:248b1fb3f739d01d528cc50b35ee9c4812aa58cc5935998e776bf8ed5b251e75", size = 11424693 }, - { url = "https://files.pythonhosted.org/packages/ac/b7/30bc78a37648d31bfc7ba7105b108cb9091cd925f249aa533038ebc5a96f/ruff-0.8.6-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:bc3c083c50390cf69e7e1b5a5a7303898966be973664ec0c4a4acea82c1d4315", size = 12141921 }, - { url = "https://files.pythonhosted.org/packages/60/b3/ee0a14cf6a1fbd6965b601c88d5625d250b97caf0534181e151504498f86/ruff-0.8.6-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:52d587092ab8df308635762386f45f4638badb0866355b2b86760f6d3c076188", size = 11692419 }, - { url = "https://files.pythonhosted.org/packages/ef/d6/c597062b2931ba3e3861e80bd2b147ca12b3370afc3889af46f29209037f/ruff-0.8.6-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:61323159cf21bc3897674e5adb27cd9e7700bab6b84de40d7be28c3d46dc67cf", size = 12981648 }, - { url = "https://files.pythonhosted.org/packages/68/84/21f578c2a4144917985f1f4011171aeff94ab18dfa5303ac632da2f9af36/ruff-0.8.6-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ae4478b1471fc0c44ed52a6fb787e641a2ac58b1c1f91763bafbc2faddc5117", size = 11251801 }, - { url = "https://files.pythonhosted.org/packages/6c/aa/1ac02537c8edeb13e0955b5db86b5c050a1dcba54f6d49ab567decaa59c1/ruff-0.8.6-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:0c000a471d519b3e6cfc9c6680025d923b4ca140ce3e4612d1a2ef58e11f11fe", size = 10849857 }, - { url = "https://files.pythonhosted.org/packages/eb/00/020cb222252d833956cb3b07e0e40c9d4b984fbb2dc3923075c8f944497d/ruff-0.8.6-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:9257aa841e9e8d9b727423086f0fa9a86b6b420fbf4bf9e1465d1250ce8e4d8d", size = 10470852 }, - { url = "https://files.pythonhosted.org/packages/00/56/e6d6578202a0141cd52299fe5acb38b2d873565f4670c7a5373b637cf58d/ruff-0.8.6-py3-none-musllinux_1_2_i686.whl", hash = "sha256:45a56f61b24682f6f6709636949ae8cc82ae229d8d773b4c76c09ec83964a95a", size = 10972997 }, - { url = "https://files.pythonhosted.org/packages/be/31/dd0db1f4796bda30dea7592f106f3a67a8f00bcd3a50df889fbac58e2786/ruff-0.8.6-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:496dd38a53aa173481a7d8866bcd6451bd934d06976a2505028a50583e001b76", size = 11317760 }, - { url = "https://files.pythonhosted.org/packages/d4/70/cfcb693dc294e034c6fed837fa2ec98b27cc97a26db5d049345364f504bf/ruff-0.8.6-py3-none-win32.whl", hash = "sha256:e169ea1b9eae61c99b257dc83b9ee6c76f89042752cb2d83486a7d6e48e8f764", size = 8799729 }, - { url = "https://files.pythonhosted.org/packages/60/22/ae6bcaa0edc83af42751bd193138bfb7598b2990939d3e40494d6c00698c/ruff-0.8.6-py3-none-win_amd64.whl", hash = "sha256:f1d70bef3d16fdc897ee290d7d20da3cbe4e26349f62e8a0274e7a3f4ce7a905", size = 9673857 }, - { url = "https://files.pythonhosted.org/packages/91/f8/3765e053acd07baa055c96b2065c7fab91f911b3c076dfea71006666f5b0/ruff-0.8.6-py3-none-win_arm64.whl", hash = "sha256:7d7fc2377a04b6e04ffe588caad613d0c460eb2ecba4c0ccbbfe2bc973cbc162", size = 9149556 }, +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/75/48/385f276f41e89623a5ea8e4eb9c619a44fdfc2a64849916b3584eca6cb9f/ruff-0.9.0.tar.gz", hash = "sha256:143f68fa5560ecf10fc49878b73cee3eab98b777fcf43b0e62d43d42f5ef9d8b", size = 3489167 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/01/e0885e5519212efc7ab9d868bc39cb9781931c4c6f9b17becafa81193ec4/ruff-0.9.0-py3-none-linux_armv6l.whl", hash = "sha256:949b3513f931741e006cf267bf89611edff04e1f012013424022add3ce78f319", size = 10647069 }, + { url = "https://files.pythonhosted.org/packages/dd/69/510a9a5781dcf84c2ad513c2003936fefc802f39c745d5f2355d77fa45fd/ruff-0.9.0-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:99fbcb8c7fe94ae1e462ab2a1ef17cb20b25fb6438b9f198b1bcf5207a0a7916", size = 10401936 }, + { url = "https://files.pythonhosted.org/packages/07/9f/37fb86bfdf28c4cbfe94cbcc01fb9ab0cb8128548f243f34d5298b212562/ruff-0.9.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0b022afd8eb0fcfce1e0adec84322abf4d6ce3cd285b3b99c4f17aae7decf749", size = 10010347 }, + { url = "https://files.pythonhosted.org/packages/30/0d/b95121f53c7f7bfb7ba427a35d25f983ed3b476620c5cd69f45caa5b294e/ruff-0.9.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:336567ce92c9ca8ec62780d07b5fa11fbc881dc7bb40958f93a7d621e7ab4589", size = 10882152 }, + { url = "https://files.pythonhosted.org/packages/d4/0b/a955cb6b19eb900c4c594707ab72132ce2d5cd8b5565137fb8fed21b8f08/ruff-0.9.0-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:d338336c44bda602dc8e8766836ac0441e5b0dfeac3af1bd311a97ebaf087a75", size = 10405502 }, + { url = "https://files.pythonhosted.org/packages/1e/fa/9a6c70af74f20edd2519b89eb3322f4bfa399315cf306383443700f2d6b6/ruff-0.9.0-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d9b3ececf523d733e90b540e7afcc0494189e8999847f8855747acd5a9a8c45f", size = 11465069 }, + { url = "https://files.pythonhosted.org/packages/ee/8b/7effac8915470da496be009fe861060baff2692f92801976b2c01cdc8c54/ruff-0.9.0-py3-none-manylinux_2_17_ppc64.manylinux2014_ppc64.whl", hash = "sha256:a11c0872a31232e473e2e0e2107f3d294dbadd2f83fb281c3eb1c22a24866924", size = 12176850 }, + { url = "https://files.pythonhosted.org/packages/bd/ed/626179786889eca47b1e821c1582622ac0c1c8f01d60ac974f8b96867a57/ruff-0.9.0-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b5fd06220c17a9cc0dc7fc6552f2ac4db74e8e8bff9c401d160ac59d00566f54", size = 11700963 }, + { url = "https://files.pythonhosted.org/packages/75/79/094c34ddec47fd3c61a0bc5e83ca164344c592949cff91f05961fd40922e/ruff-0.9.0-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:0457e775c74bf3976243f910805242b7dcd389e1d440deccbd1194ca17a5728c", size = 13096560 }, + { url = "https://files.pythonhosted.org/packages/e7/23/ec85dca0dcb329835197401734501bfa1d39e72343df64628c67b72bcbf5/ruff-0.9.0-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:05415599bbcb318f730ea1b46a39e4fbf71f6a63fdbfa1dda92efb55f19d7ecf", size = 11278658 }, + { url = "https://files.pythonhosted.org/packages/6c/17/1b3ea5f06578ea1daa08ac35f9de099d1827eea6e116a8cabbf11235c925/ruff-0.9.0-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:fbf9864b009e43cfc1c8bed1a6a4c529156913105780af4141ca4342148517f5", size = 10879847 }, + { url = "https://files.pythonhosted.org/packages/a6/e5/00bc97d6f419da03c0d898e95cca77311494e7274dc7cc17d94976e32e52/ruff-0.9.0-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:37b3da222b12e2bb2ce628e02586ab4846b1ed7f31f42a5a0683b213453b2d49", size = 10494220 }, + { url = "https://files.pythonhosted.org/packages/cc/70/d0a23d94f3e40b7ffac0e5506f33bb504672569173781a6c7cab0db6a4ba/ruff-0.9.0-py3-none-musllinux_1_2_i686.whl", hash = "sha256:733c0fcf2eb0c90055100b4ed1af9c9d87305b901a8feb6a0451fa53ed88199d", size = 11004182 }, + { url = "https://files.pythonhosted.org/packages/20/8e/367cf8e401890f823d0e4eb33635d0113719d5660b6522b7295376dd95fd/ruff-0.9.0-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:8221a454bfe5ccdf8017512fd6bb60e6ec30f9ea252b8a80e5b73619f6c3cefd", size = 11345761 }, + { url = "https://files.pythonhosted.org/packages/fe/08/4b54e02da73060ebc29368ab15868613f7d2496bde3b01d284d5423646bc/ruff-0.9.0-py3-none-win32.whl", hash = "sha256:d345f2178afd192c7991ddee59155c58145e12ad81310b509bd2e25c5b0247b3", size = 8807005 }, + { url = "https://files.pythonhosted.org/packages/a1/a7/0b422971e897c51bf805f998d75bcfe5d4d858f5002203832875fc91b733/ruff-0.9.0-py3-none-win_amd64.whl", hash = "sha256:0cbc0905d94d21305872f7f8224e30f4bbcd532bc21b2225b2446d8fc7220d19", size = 9689974 }, + { url = "https://files.pythonhosted.org/packages/73/0e/c00f66731e514be3299801b1d9d54efae0abfe8f00a5c14155f2ab9e2920/ruff-0.9.0-py3-none-win_arm64.whl", hash = "sha256:7b1148771c6ca88f820d761350a053a5794bc58e0867739ea93eb5e41ad978cd", size = 9147729 }, ] [[package]] @@ -4783,6 +4774,7 @@ dapr = [ ] google = [ { name = "google-cloud-aiplatform", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "google-genai", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "google-generativeai", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] hugging-face = [ @@ -4872,6 +4864,7 @@ requires-dist = [ { name = "defusedxml", specifier = "~=0.7" }, { name = "flask-dapr", marker = "extra == 'dapr'", specifier = ">=1.14.0" }, { name = "google-cloud-aiplatform", marker = "extra == 'google'", specifier = "~=1.60" }, + { name = "google-genai", marker = "extra == 'google'", specifier = "~=0.4" }, { name = "google-generativeai", marker = "extra == 'google'", specifier = "~=0.7" }, { name = "ipykernel", marker = "extra == 'notebooks'", specifier = "~=6.29" }, { name = "jinja2", specifier = "~=3.1" }, @@ -5440,11 +5433,11 @@ wheels = [ [[package]] name = "types-setuptools" -version = "75.6.0.20241223" +version = "75.8.0.20250110" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/53/48/a89068ef20e3bbb559457faf0fd3c18df6df5df73b4b48ebf466974e1f54/types_setuptools-75.6.0.20241223.tar.gz", hash = "sha256:d9478a985057ed48a994c707f548e55aababa85fe1c9b212f43ab5a1fffd3211", size = 48063 } +sdist = { url = "https://files.pythonhosted.org/packages/f7/42/5713e90d4f9683f2301d900f33e4fc2405ad8ac224dda30f6cb7f4cd215b/types_setuptools-75.8.0.20250110.tar.gz", hash = "sha256:96f7ec8bbd6e0a54ea180d66ad68ad7a1d7954e7281a710ea2de75e355545271", size = 48185 } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/2f/051d5d23711209d4077d95c62fa8ef6119df7298635e3a929e50376219d1/types_setuptools-75.6.0.20241223-py3-none-any.whl", hash = "sha256:7cbfd3bf2944f88bbcdd321b86ddd878232a277be95d44c78a53585d78ebc2f6", size = 71377 }, + { url = "https://files.pythonhosted.org/packages/cf/a3/dbfd106751b11c728cec21cc62cbfe7ff7391b935c4b6e8f0bdc2e6fd541/types_setuptools-75.8.0.20250110-py3-none-any.whl", hash = "sha256:a9f12980bbf9bcdc23ecd80755789085bad6bfce4060c2275bc2b4ca9f2bc480", size = 71521 }, ] [[package]] From a3df625560d2a05bd1fd7f56a1a235149a69057b Mon Sep 17 00:00:00 2001 From: Eduard van Valkenburg Date: Fri, 10 Jan 2025 16:59:35 +0100 Subject: [PATCH 06/25] small updates --- docs/decisions/00XX-realtime-api-clients.md | 90 ++++++++++--------- .../audio/04-chat_with_realtime_api.py | 9 +- 2 files changed, 51 insertions(+), 48 deletions(-) diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index 81d9d6fdf4e7..8dff5f882cf6 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -23,51 +23,51 @@ One feature that we need to consider if and how to deal with is whether or not a ### Event types Client side events: -| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | -|-------------------| ------------------------------------|-------------------------|------------------------| - | Control | Configure session | `session.update` | `BidiGenerateContentSetup` | - | Content | Send voice input | `input_audio_buffer.append` | `BidiGenerateContentRealtimeInput` | - | Control | Commit input and request response | `input_audio_buffer.commit` | `-` | - | Control | Clean audio input buffer | `input_audio_buffer.clear` | `-` | - | Content | Send text input | `conversation.item.create` | `BidiGenerateContentClientContent` | - | Control | Interrupt audio | `conversation.item.truncate` | `-`| - | Control | Delete content | `conversation.item.delete` | `-`| -| Control | Respond to function call request | `conversation.item.create` | `BidiGenerateContentToolResponse`| -| Control | Ask for response | `response.create` | `-`| -| Control | Cancel response | `response.cancel` | `-`| +| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | +| ------------------------- | --------------------------------- | ---------------------------- | ---------------------------------- | +| Control | Configure session | `session.update` | `BidiGenerateContentSetup` | +| Content | Send voice input | `input_audio_buffer.append` | `BidiGenerateContentRealtimeInput` | +| Control | Commit input and request response | `input_audio_buffer.commit` | `-` | +| Control | Clean audio input buffer | `input_audio_buffer.clear` | `-` | +| Content | Send text input | `conversation.item.create` | `BidiGenerateContentClientContent` | +| Control | Interrupt audio | `conversation.item.truncate` | `-` | +| Control | Delete content | `conversation.item.delete` | `-` | +| Control | Respond to function call request | `conversation.item.create` | `BidiGenerateContentToolResponse` | +| Control | Ask for response | `response.create` | `-` | +| Control | Cancel response | `response.cancel` | `-` | Server side events: -| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | -|----------------------------|-------------------------------------|-------------------------|------------------------| -| Control | Error | `error` | `-` | -| Control | Session created | `session.created` | `BidiGenerateContentSetupComplete` | -| Control | Session updated | `session.updated` | `BidiGenerateContentSetupComplete` | -| Control | Conversation created | `conversation.created` | `-` | -| Control | Input audio buffer committed | `input_audio_buffer.committed` | `-` | -| Control | Input audio buffer cleared | `input_audio_buffer.cleared` | `-` | -| Control | Input audio buffer speech started | `input_audio_buffer.speech_started` | `-` | -| Control | Input audio buffer speech stopped | `input_audio_buffer.speech_stopped` | `-` | -| Content | Conversation item created | `conversation.item.created` | `-` | -| Content | Input audio transcription completed | `conversation.item.input_audio_transcription.completed` | -| Content | Input audio transcription failed | `conversation.item.input_audio_transcription.failed` | -| Control | Conversation item truncated | `conversation.item.truncated` | `-` | -| Control | Conversation item deleted | `conversation.item.deleted` | `-` | -| Control | Response created | `response.created` | `-` | -| Control | Response done | `response.done` | `-` | -| Content | Response output item added | `response.output_item.added` | `-` | -| Content | Response output item done | `response.output_item.done` | `-` | -| Content | Response content part added | `response.content_part.added` | `-` | -| Content | Response content part done | `response.content_part.done` | `-` | -| Content | Response text delta | `response.text.delta` | `BidiGenerateContentServerContent` | -| Content | Response text done | `response.text.done` | `-` | -| Content | Response audio transcript delta | `response.audio_transcript.delta` | `BidiGenerateContentServerContent` | -| Content | Response audio transcript done | `response.audio_transcript.done` | `-` | -| Content | Response audio delta | `response.audio.delta` | `BidiGenerateContentServerContent` | -| Content | Response audio done | `response.audio.done` | `-` | -| Content | Response function call arguments delta | `response.function_call_arguments.delta` | `BidiGenerateContentToolCall` | -| Content | Response function call arguments done | `response.function_call_arguments.done` | `-` | -| Control | Function call cancelled | `-` | `BidiGenerateContentToolCallCancellation` | -| Control | Rate limits updated | `rate_limits.updated` | `-` | +| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | +| ------------------------- | -------------------------------------- | ------------------------------------------------------- | ----------------------------------------- | +| Control | Error | `error` | `-` | +| Control | Session created | `session.created` | `BidiGenerateContentSetupComplete` | +| Control | Session updated | `session.updated` | `BidiGenerateContentSetupComplete` | +| Control | Conversation created | `conversation.created` | `-` | +| Control | Input audio buffer committed | `input_audio_buffer.committed` | `-` | +| Control | Input audio buffer cleared | `input_audio_buffer.cleared` | `-` | +| Control | Input audio buffer speech started | `input_audio_buffer.speech_started` | `-` | +| Control | Input audio buffer speech stopped | `input_audio_buffer.speech_stopped` | `-` | +| Content | Conversation item created | `conversation.item.created` | `-` | +| Content | Input audio transcription completed | `conversation.item.input_audio_transcription.completed` | +| Content | Input audio transcription failed | `conversation.item.input_audio_transcription.failed` | +| Control | Conversation item truncated | `conversation.item.truncated` | `-` | +| Control | Conversation item deleted | `conversation.item.deleted` | `-` | +| Control | Response created | `response.created` | `-` | +| Control | Response done | `response.done` | `-` | +| Content | Response output item added | `response.output_item.added` | `-` | +| Content | Response output item done | `response.output_item.done` | `-` | +| Content | Response content part added | `response.content_part.added` | `-` | +| Content | Response content part done | `response.content_part.done` | `-` | +| Content | Response text delta | `response.text.delta` | `BidiGenerateContentServerContent` | +| Content | Response text done | `response.text.done` | `-` | +| Content | Response audio transcript delta | `response.audio_transcript.delta` | `BidiGenerateContentServerContent` | +| Content | Response audio transcript done | `response.audio_transcript.done` | `-` | +| Content | Response audio delta | `response.audio.delta` | `BidiGenerateContentServerContent` | +| Content | Response audio done | `response.audio.done` | `-` | +| Content | Response function call arguments delta | `response.function_call_arguments.delta` | `BidiGenerateContentToolCall` | +| Content | Response function call arguments done | `response.function_call_arguments.done` | `-` | +| Control | Function call cancelled | `-` | `BidiGenerateContentToolCallCancellation` | +| Control | Rate limits updated | `rate_limits.updated` | `-` | @@ -77,6 +77,7 @@ Server side events: - Simple programming model that is likely able to handle future realtime api's and evolution of the existing ones. - Support for the most common scenario's and content, extensible for the rest. - Natively integrated with Semantic Kernel especially for content types and function calling. +- Support multiple types of connections, like websocket and WebRTC - … @@ -94,8 +95,9 @@ This would mean there are two mechanisms in the clients, one deals with content, - Pro: - strongly typed responses for known content - easy to use as the main interactions are clear with familiar SK content types, the rest goes through a separate mechanism + - this might fit better with something like WebRTC that has distinct channels for audio and video vs a data stream for all other events - Con: - - new content support requires updates in the codebase and can be considered breaking (potentitally sending additional types back) + - new content support requires updates in the codebase and can be considered breaking (potentially sending additional types back) - additional complexity in dealing with two streams of data ### Treat everything as content items diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index bffbad691716..3aa2a8f52b10 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -47,7 +47,7 @@ def check_audio_devices(): print(sd.query_devices()) -# check_audio_devices() +check_audio_devices() class Speaker: @@ -106,6 +106,7 @@ def __init__(self, audio_recorder: AudioRecorderStream, realtime_client: Realtim self.realtime_client = realtime_client async def record_audio(self): + await self.realtime_client.send_event("response.create") with contextlib.suppress(asyncio.CancelledError): async for content in self.audio_recorder.stream_audio_content(): if content.data: @@ -150,8 +151,8 @@ async def main() -> None: realtime_client.register_event_handler("response.created", response_created_callback) # create the speaker and microphone - speaker = Speaker(AudioPlayerAsync(device_id=7), realtime_client, kernel) - microphone = Microphone(AudioRecorderStream(device_id=2), realtime_client) + speaker = Speaker(AudioPlayerAsync(device_id=None), realtime_client, kernel) + microphone = Microphone(AudioRecorderStream(device_id=None), realtime_client) # Create the settings for the session # the key thing to decide on is to enable the server_vad turn detection @@ -186,7 +187,7 @@ async def main() -> None: if __name__ == "__main__": print( - "Instruction: start speaking, when you stop the API should detect you finished and start responding." + "Instruction: start speaking, when you stop the API should detect you finished and start responding. " "Press ctrl + c to stop the program." ) asyncio.run(main()) From ab8c0826223ee222083689c2635238cb067bdb7d Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Tue, 14 Jan 2025 15:45:04 +0100 Subject: [PATCH 07/25] webrtc WIP --- docs/decisions/00XX-realtime-api-clients.md | 23 +- python/pyproject.toml | 1 + .../audio/04-chat_with_realtime_api.py | 121 ++++-- .../concepts/audio/audio_player_async.py | 4 +- .../concepts/audio/audio_recorder_stream.py | 23 +- .../connectors/ai/open_ai/__init__.py | 3 +- .../ai/open_ai/services/open_ai_realtime.py | 59 ++- .../open_ai/services/open_ai_realtime_base.py | 405 +++++++++++++++++- .../connectors/ai/realtime_client_base.py | 74 ++-- .../semantic_kernel/contents/audio_content.py | 6 + .../contents/binary_content.py | 39 +- .../contents/utils/data_uri.py | 27 +- python/uv.lock | 69 +-- 13 files changed, 701 insertions(+), 153 deletions(-) diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index 8dff5f882cf6..1b0bbd2d6c52 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -14,13 +14,21 @@ informed: Multiple model providers are starting to enable realtime voice-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. The key addition that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. -The way these API's work at this time is through either websockets or WebRTC. In both cases there are events being sent to and from the service, some events contain content, text, audio, or video (so far only sending, not receiving), while some events are "control" events, like content created, function call requested, etc. Sending events include, sending content, either voice, text or function call output, or events, like committing the input audio and requesting a response. +The way these API's work at this time is through either Websockets or WebRTC. + +In both cases there are events being sent to and from the service, some events contain content, text, audio, or video (so far only sending, not receiving), while some events are "control" events, like content created, function call requested, etc. Sending events include, sending content, either voice, text or function call output, or events, like committing the input audio and requesting a response. + +### Websocket +Websocket has been around for a while and is a well known technology, it is a full-duplex communication protocol over a single, long-lived connection. It is used for sending and receiving messages between client and server in real-time. Each event can contain a message, which might contain a content item, or a control event. + +### WebRTC +WebRTC is a Mozilla project that provides web browsers and mobile applications with real-time communication via simple application programming interfaces (APIs). It allows audio and video communication to work inside web pages by allowing direct peer-to-peer communication, eliminating the need to install plugins or download native apps. It is used for sending and receiving audio and video streams, and can be used for sending messages as well. The big difference compared to websockets is that it does explicitly create a channel for audio and video, and a separate channel for "data", which are events but also things like Function calls. Both the OpenAI and Google realtime api's are in preview/beta, this means there might be breaking changes in the way they work coming in the future, therefore the clients built to support these API's are going to be experimental until the API's stabilize. One feature that we need to consider if and how to deal with is whether or not a service uses Voice Activated Detection, OpenAI supports turning that off and allows parameters for how it behaves, while Google has it on by default and it cannot be configured. -### Event types +### Event types (websocket and partially webrtc) Client side events: | **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | @@ -48,8 +56,8 @@ Server side events: | Control | Input audio buffer speech started | `input_audio_buffer.speech_started` | `-` | | Control | Input audio buffer speech stopped | `input_audio_buffer.speech_stopped` | `-` | | Content | Conversation item created | `conversation.item.created` | `-` | -| Content | Input audio transcription completed | `conversation.item.input_audio_transcription.completed` | -| Content | Input audio transcription failed | `conversation.item.input_audio_transcription.failed` | +| Content | Input audio transcription completed | `conversation.item.input_audio_transcription.completed` | | +| Content | Input audio transcription failed | `conversation.item.input_audio_transcription.failed` | | | Control | Conversation item truncated | `conversation.item.truncated` | `-` | | Control | Conversation item deleted | `conversation.item.deleted` | `-` | | Control | Response created | `response.created` | `-` | @@ -70,16 +78,15 @@ Server side events: | Control | Rate limits updated | `rate_limits.updated` | `-` | - - ## Decision Drivers - Simple programming model that is likely able to handle future realtime api's and evolution of the existing ones. - Support for the most common scenario's and content, extensible for the rest. - Natively integrated with Semantic Kernel especially for content types and function calling. - Support multiple types of connections, like websocket and WebRTC - -- … + +## Decision driver questions +- For WebRTC, a audio device can be passed, should this be a requirement for the client also for websockets? ## Considered Options diff --git a/python/pyproject.toml b/python/pyproject.toml index b5efc0723cf3..67b762bfd78d 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -45,6 +45,7 @@ dependencies = [ "pybars4 ~= 0.9", "jinja2 ~= 3.1", "nest-asyncio ~= 1.6", + "taskgroup >= 0.2.2; python_version < '3.11'", ] ### Optional dependencies diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index 3aa2a8f52b10..40f16a2c0a24 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -5,6 +5,9 @@ import signal from typing import Any +import numpy as np +from aiortc.mediastreams import MediaStreamError, MediaStreamTrack +from av import AudioFrame from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent from samples.concepts.audio.audio_player_async import AudioPlayerAsync @@ -12,8 +15,8 @@ from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai import ( - OpenAIRealtime, OpenAIRealtimeExecutionSettings, + OpenAIRealtimeWebRTC, TurnDetection, ) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings @@ -69,52 +72,77 @@ async def play( ) -> None: # reset the frame count for the audio player self.audio_player.reset_frame_count() - # open the connection to the realtime api - async with self.realtime_client as client: - # update the session with the chat_history and settings - await client.update_session(settings=settings, chat_history=chat_history) - # print the start message of the transcript - if print_transcript: - print("Mosscap (transcript): ", end="") - try: - # start listening for events - async for content in self.realtime_client.event_listener(settings=settings, kernel=self.kernel): - if not content: - continue - # the contents returned should be StreamingChatMessageContent - # so we will loop through the items within it. - for item in content.items: - match item: - case StreamingTextContent(): - if print_transcript: - print(item.text, end="") - await asyncio.sleep(0.01) - continue - case AudioContent(): - self.audio_player.add_data(item.data) - await asyncio.sleep(0.01) - continue - except asyncio.CancelledError: - print("\nThanks for talking to Mosscap!") - - -class Microphone: + # print the start message of the transcript + if print_transcript: + print("Mosscap (transcript): ", end="") + try: + # start listening for events + while True: + _, content = await self.realtime_client.output_buffer.get() + if not content: + continue + # the contents returned should be StreamingChatMessageContent + # so we will loop through the items within it. + for item in content.items: + match item: + case StreamingTextContent(): + if print_transcript: + print(item.text, end="") + await asyncio.sleep(0.01) + continue + case AudioContent(): + self.audio_player.add_data(item.data) + await asyncio.sleep(0.01) + continue + except asyncio.CancelledError: + print("\nThanks for talking to Mosscap!") + + +class Microphone(MediaStreamTrack): """This is a simple class that opens the microphone and sends the audio to the realtime api.""" + kind = "audio" + def __init__(self, audio_recorder: AudioRecorderStream, realtime_client: RealtimeClientBase): self.audio_recorder = audio_recorder self.realtime_client = realtime_client + self.queue = asyncio.Queue() + self.loop = asyncio.get_running_loop() + self._pts = 0 + + async def recv(self) -> Any: + # start the audio recording + try: + return await self.queue.get() + except Exception as e: + logger.error(f"Error receiving audio frame: {str(e)}") + raise MediaStreamError("Failed to receive audio frame") async def record_audio(self): - await self.realtime_client.send_event("response.create") - with contextlib.suppress(asyncio.CancelledError): - async for content in self.audio_recorder.stream_audio_content(): - if content.data: - await self.realtime_client.send_event( - "input_audio_buffer.append", - content=content, - ) - await asyncio.sleep(0.01) + def callback(indata, frames, time, status): + if status: + logger.warning(f"Audio input status: {status}") + audio_data = indata.copy() + + if audio_data.dtype != np.int16: + audio_data = (audio_data * 32767).astype(np.int16) + + # Create AudioFrame with incrementing pts + frame = AudioFrame( + samples=len(audio_data), + layout="mono", + format="s16", # 16-bit signed integer + ) + frame.rate = 48000 + frame.pts = self._pts + self._pts += len(audio_data) # Increment pts by frame size + + frame.planes[0].update(audio_data.tobytes()) + + asyncio.run_coroutine_threadsafe(self.queue.put(frame), self.loop) + + await self.realtime_client.input_buffer.put("response.create") + await self.audio_recorder.stream_audio_content_with_callback(callback=callback) # this function is used to stop the processes when ctrl + c is pressed @@ -147,7 +175,7 @@ async def main() -> None: kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) # create the realtime client and register the response created callback - realtime_client = OpenAIRealtime(ai_model_id="gpt-4o-realtime-preview-2024-12-17") + realtime_client = OpenAIRealtimeWebRTC(ai_model_id="gpt-4o-realtime-preview-2024-12-17") realtime_client.register_event_handler("response.created", response_created_callback) # create the speaker and microphone @@ -180,9 +208,14 @@ async def main() -> None: turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), function_choice_behavior=FunctionChoiceBehavior.Auto(), ) - # start the the speaker and the microphone - with contextlib.suppress(asyncio.CancelledError): - await asyncio.gather(*[speaker.play(chat_history, settings), microphone.record_audio()]) + async with realtime_client: + await realtime_client.update_session(settings=settings, chat_history=chat_history) + await realtime_client.start_listening(settings, chat_history) + await realtime_client.start_sending(input_audio_track=microphone) + # await realtime_client.start_streaming(settings, chat_history, input_audio_track=microphone) + # start the the speaker and the microphone + with contextlib.suppress(asyncio.CancelledError): + await speaker.play(chat_history, settings) if __name__ == "__main__": diff --git a/python/samples/concepts/audio/audio_player_async.py b/python/samples/concepts/audio/audio_player_async.py index a77b8df6e32c..36c1492094a6 100644 --- a/python/samples/concepts/audio/audio_player_async.py +++ b/python/samples/concepts/audio/audio_player_async.py @@ -53,10 +53,10 @@ def reset_frame_count(self): def get_frame_count(self): return self._frame_count - def add_data(self, data: bytes): + def add_data(self, data: bytes | np.ndarray): with self.lock: # bytes is pcm16 single channel audio data, convert to numpy array - np_data = np.frombuffer(data, dtype=np.int16) + np_data = np.frombuffer(data, dtype=np.int16) if isinstance(data, bytes) else data self.queue.append(np_data) if not self.playing: self.start() diff --git a/python/samples/concepts/audio/audio_recorder_stream.py b/python/samples/concepts/audio/audio_recorder_stream.py index 55684e9c469b..20c758af3e39 100644 --- a/python/samples/concepts/audio/audio_recorder_stream.py +++ b/python/samples/concepts/audio/audio_recorder_stream.py @@ -2,9 +2,10 @@ import asyncio import base64 -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Callable from typing import Any, ClassVar, cast +import sounddevice as sd from pydantic import BaseModel from semantic_kernel.contents.audio_content import AudioContent @@ -30,9 +31,25 @@ class AudioRecorderStream(BaseModel): CHUNK_LENGTH_S: ClassVar[float] = 0.05 device_id: int | None = None - async def stream_audio_content(self) -> AsyncGenerator[AudioContent, None]: - import sounddevice as sd # type: ignore + async def stream_audio_content_with_callback(self, callback: Callable[..., Any]) -> None: + stream = sd.InputStream( + channels=self.CHANNELS, + samplerate=self.SAMPLE_RATE, + dtype="int16", + device=self.device_id, + callback=callback, + ) + stream.start() + try: + while True: + await asyncio.sleep(0) + except KeyboardInterrupt: + pass + finally: + stream.stop() + stream.close() + async def stream_audio_content(self) -> AsyncGenerator[AudioContent, None]: # device_info = sd.query_devices() # print(device_info) diff --git a/python/semantic_kernel/connectors/ai/open_ai/__init__.py b/python/semantic_kernel/connectors/ai/open_ai/__init__.py index 27d36ea30d34..2c2a87a64a7b 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/__init__.py +++ b/python/semantic_kernel/connectors/ai/open_ai/__init__.py @@ -40,7 +40,7 @@ from semantic_kernel.connectors.ai.open_ai.services.azure_text_to_image import AzureTextToImage from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text import OpenAIAudioToText from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletion -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime, OpenAIRealtimeWebRTC from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding import OpenAITextEmbedding from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio import OpenAITextToAudio @@ -76,6 +76,7 @@ "OpenAIPromptExecutionSettings", "OpenAIRealtime", "OpenAIRealtimeExecutionSettings", + "OpenAIRealtimeWebRTC", "OpenAISettings", "OpenAITextCompletion", "OpenAITextEmbedding", diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index 23351d7b6176..39c85816ced3 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -7,7 +7,10 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_base import OpenAIRealtimeBase +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_base import ( + OpenAIRealtimeBase, + OpenAIRealtimeWebRTCBase, +) from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError @@ -64,3 +67,57 @@ def __init__( default_headers=default_headers, client=async_client, ) + + +class OpenAIRealtimeWebRTC(OpenAIRealtimeWebRTCBase, OpenAIConfigBase): + """OpenAI Realtime service.""" + + def __init__( + self, + ai_model_id: str | None = None, + api_key: str | None = None, + org_id: str | None = None, + service_id: str | None = None, + default_headers: Mapping[str, str] | None = None, + async_client: AsyncOpenAI | None = None, + env_file_path: str | None = None, + env_file_encoding: str | None = None, + ) -> None: + """Initialize an OpenAITextCompletion service. + + Args: + ai_model_id (str | None): OpenAI model name, see + https://platform.openai.com/docs/models + service_id (str | None): Service ID tied to the execution settings. + api_key (str | None): The optional API key to use. If provided will override, + the env vars or .env file value. + org_id (str | None): The optional org ID to use. If provided will override, + the env vars or .env file value. + default_headers: The default headers mapping of string keys to + string values for HTTP requests. (Optional) + async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional) + env_file_path (str | None): Use the environment settings file as a fallback to + environment variables. (Optional) + env_file_encoding (str | None): The encoding of the environment settings file. (Optional) + """ + try: + openai_settings = OpenAISettings.create( + api_key=api_key, + org_id=org_id, + text_model_id=ai_model_id, + env_file_path=env_file_path, + env_file_encoding=env_file_encoding, + ) + except ValidationError as ex: + raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex + if not openai_settings.text_model_id: + raise ServiceInitializationError("The OpenAI text model ID is required.") + super().__init__( + ai_model_id=openai_settings.text_model_id, + service_id=service_id, + api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None, + org_id=openai_settings.org_id, + ai_model_type=OpenAIModelTypes.TEXT, + default_headers=default_headers, + client=async_client, + ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py index 64b647f44ee8..f82bce19164f 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -2,6 +2,7 @@ import asyncio import base64 +import json import logging import sys from collections.abc import AsyncGenerator @@ -14,6 +15,15 @@ else: from typing_extensions import override # pragma: no cover +from aiohttp import ClientSession +from aiortc import ( + MediaStreamTrack, + RTCConfiguration, + RTCDataChannel, + RTCIceServer, + RTCPeerConnection, + RTCSessionDescription, +) from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection from openai.types.beta.realtime.conversation_item_create_event_param import ConversationItemParam from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent @@ -152,7 +162,7 @@ def register_event_handler( self.event_handlers.setdefault(event_type, []).append(handler) @override - async def event_listener( + async def start_listening( self, settings: "PromptExecutionSettings", chat_history: "ChatHistory | None" = None, @@ -186,7 +196,7 @@ async def event_listener( logger.debug(f"Event type: {event_type}, count: {len(self.event_log[event_type])}") @override - async def send_event(self, event: str | SendEvents, **kwargs: Any) -> None: + async def start_sending(self, event: str | SendEvents, **kwargs: Any) -> None: await self.connected.wait() if not self.connection: raise ValueError("Connection is not established.") @@ -299,10 +309,10 @@ async def update_session( self._update_function_choice_settings_callback(), kernel=kwargs.get("kernel"), # type: ignore ) - await self.send_event(SendEvents.SESSION_UPDATE, settings=settings) + await self.start_sending(SendEvents.SESSION_UPDATE, settings=settings) if chat_history and len(chat_history) > 0: await asyncio.gather( - *(self.send_event(SendEvents.CONVERSATION_ITEM_CREATE, item=msg) for msg in chat_history.messages) + *(self.start_sending(SendEvents.CONVERSATION_ITEM_CREATE, item=msg) for msg in chat_history.messages) ) @override @@ -313,6 +323,8 @@ async def close_session(self) -> None: self.connection = None self.connected.clear() + # region Event callbacks + def response_audio_delta_callback( self, event: RealtimeServerEvent, @@ -420,13 +432,392 @@ async def response_function_call_arguments_done_callback( if kernel: chat_history = ChatHistory() await kernel.invoke_function_call(item, chat_history) - await self.send_event(SendEvents.CONVERSATION_ITEM_CREATE, item=chat_history.messages[-1]) + await self.start_sending(SendEvents.CONVERSATION_ITEM_CREATE, item=chat_history.messages[-1]) + # The model doesn't start responding to the tool call automatically, so triggering it here. + await self.start_sending(SendEvents.RESPONSE_CREATE) + return chat_history.messages[-1], False + + # region settings + + @override + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: + from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa + OpenAIRealtimeExecutionSettings, + ) + + return OpenAIRealtimeExecutionSettings + + +@experimental_class +class OpenAIRealtimeWebRTCBase(OpenAIHandler, RealtimeClientBase): + """OpenAI WebRTC Realtime service.""" + + SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True + peer_connection: RTCPeerConnection | None = None + data_channel: RTCDataChannel | None = None + connection: AsyncRealtimeConnection | None = None + connected: asyncio.Event = Field(default_factory=asyncio.Event) + event_log: dict[str, list[RealtimeServerEvent]] = Field(default_factory=dict) + event_handlers: dict[str, list[EventCallBackProtocol | EventCallBackProtocolAsync]] = Field(default_factory=dict) + + def model_post_init(self, *args, **kwargs) -> None: + """Post init method for the model.""" + # Register the default event handlers + self.register_event_handler( + ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA, self.response_audio_transcript_delta_callback + ) + self.register_event_handler( + ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE, self.response_audio_transcript_done_callback + ) + self.register_event_handler( + ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, self.response_function_call_arguments_delta_callback + ) + self.register_event_handler(ListenEvents.ERROR, self.error_callback) + self.register_event_handler(ListenEvents.SESSION_CREATED, self.session_callback) + self.register_event_handler(ListenEvents.SESSION_UPDATED, self.session_callback) + + def register_event_handler( + self, event_type: str | ListenEvents, handler: EventCallBackProtocol | EventCallBackProtocolAsync + ) -> None: + """Register a event handler.""" + if not isinstance(event_type, ListenEvents): + event_type = ListenEvents(event_type) + self.event_handlers.setdefault(event_type, []).append(handler) + + @override + async def start_listening( + self, + settings: "PromptExecutionSettings", + chat_history: "ChatHistory | None" = None, + **kwargs: Any, + ) -> AsyncGenerator[StreamingChatMessageContent, Any]: + ice_servers = [RTCIceServer(urls=["stun:stun.l.google.com:19302"])] + self.peer_connection = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) + + @self.peer_connection.on("track") + async def on_track(track: MediaStreamTrack) -> None: + if track.kind == "audio": + while True: + frame = await track.recv() + await self.output_buffer.put( + ( + ListenEvents.RESPONSE_AUDIO_DELTA, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[AudioContent(data=frame.to_ndarray(), data_format="base64")], + choice_index=0, + inner_content=frame, + ), + ), + ) + + data_channel = self.peer_connection.createDataChannel("oai-events") + + @data_channel.on("message") + async def on_data(data: bytes) -> None: + event = RealtimeServerEvent.model_validate_strings(data) + event_type = ListenEvents(event.type) + self.event_log.setdefault(event_type, []).append(event) + for handler in self.event_handlers.get(event_type, []): + task = handler(event=event, settings=settings) + if not task: + continue + if isawaitable(task): + async_result = await task + if not async_result: + continue + result, should_return = async_result + else: + result, should_return = task + if should_return: + yield result + else: + chat_history.add_message(result) + + offer = await self.peer_connection.createOffer() + await self.peer_connection.setLocalDescription(offer) + + try: + ephemeral_token = await self.get_ephemeral_token() + headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"} + + async with ( + ClientSession() as session, + session.post( + f"{self.client.beta.realtime._client.base_url}/realtime/sessions?model={self.ai_model_id}", + headers=headers, + data=offer.sdp, + ) as response, + ): + if response.status not in [200, 201]: + error_text = await response.text() + raise Exception(f"OpenAI WebRTC error: {error_text}") + + sdp_answer = await response.text() + answer = RTCSessionDescription(sdp=sdp_answer, type="answer") + await self.peer_connection.setRemoteDescription(answer) + + except Exception as e: + logger.error(f"Failed to connect to OpenAI: {e!s}") + raise + + @override + async def start_sending(self, input_audio_track: MediaStreamTrack | None = None, **kwargs: Any) -> None: + if input_audio_track: + if not self.peer_connection: + raise ValueError("Peer connection is not established.") + self.peer_connection.addTransceiver(input_audio_track) + + if not self.data_channel: + raise ValueError("Data channel is not established.") + while True: + item = await self.input_buffer.get() + if not item: + continue + if isinstance(item, tuple): + event, data = item + else: + event = item + data = None + if not isinstance(event, SendEvents): + event = SendEvents(event) + response: dict[str, Any] = { + "type": event, + } + match event: + case SendEvents.SESSION_UPDATE: + if "settings" not in data: + logger.error("Event data does not contain 'settings'") + response["session"] = data["settings"].prepare_settings_dict() + case SendEvents.CONVERSATION_ITEM_CREATE: + if "item" not in data: + logger.error("Event data does not contain 'item'") + return + content = data["item"] + for item in content.items: + match item: + case TextContent(): + response["item"] = ConversationItemParam( + type="message", + content=[ + { + "type": "input_text", + "text": item.text, + } + ], + role="user", + ) + + case FunctionCallContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function call needs to have a call_id") + continue + response["item"] = ConversationItemParam( + type="function_call", + name=item.name, + arguments=item.arguments, + call_id=call_id, + ) + + case FunctionResultContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function result needs to have a call_id") + continue + response["item"] = ConversationItemParam( + type="function_call_output", + output=item.result, + call_id=call_id, + ) + + case SendEvents.CONVERSATION_ITEM_TRUNCATE: + if "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + response["item_id"] = data["item_id"] + response["content_index"] = 0 + response["audio_end_ms"] = data.get("audio_end_ms", 0) + + case SendEvents.CONVERSATION_ITEM_DELETE: + if "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + response["item_id"] = data["item_id"] + case SendEvents.RESPONSE_CREATE: + if "response" in data: + response["response"] = data["response"] + case SendEvents.RESPONSE_CANCEL: + if "response_id" in data: + response["response_id"] = data["response_id"] + + self.data_channel.send(json.dumps(response)) + + @override + async def create_session( + self, + settings: PromptExecutionSettings | None = None, + chat_history: ChatHistory | None = None, + **kwargs: Any, + ) -> None: + """Create a session in the service.""" + if settings or chat_history or kwargs: + await self.update_session(settings=settings, chat_history=chat_history, **kwargs) + + async def get_ephemeral_token(self) -> str: + """Get an ephemeral token from OpenAI.""" + headers = {"Authorization": f"Bearer {self.client.api_key}", "Content-Type": "application/json"} + data = {"model": self.ai_model_id, "voice": "echo"} + + try: + async with ( + ClientSession() as session, + session.post( + f"{self.client.beta.realtime._client.base_url}/realtime/sessions", headers=headers, json=data + ) as response, + ): + if response.status not in [200, 201]: + error_text = await response.text() + raise Exception(f"Failed to get ephemeral token: {error_text}") + + result = await response.json() + return result["client_secret"]["value"] + + except Exception as e: + logger.error(f"Failed to get ephemeral token: {e!s}") + raise + + @override + async def update_session( + self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any + ) -> None: + if settings: + if "kernel" in kwargs: + settings = prepare_settings_for_function_calling( + settings, + self.get_prompt_execution_settings_class(), + self._update_function_choice_settings_callback(), + kernel=kwargs.get("kernel"), # type: ignore + ) + await self.input_buffer.put((SendEvents.SESSION_UPDATE, {"settings": settings})) + if chat_history and len(chat_history) > 0: + for msg in chat_history.messages: + await self.input_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": msg})) + + @override + async def close_session(self) -> None: + """Close the session in the service.""" + if self.peer_connection: + await self.peer_connection.close() + if self.data_channel: + await self.data_channel.close() + self.peer_connection = None + self.data_channel = None + + # region Event callbacks + + def response_audio_transcript_delta_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response audio transcript delta.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[StreamingTextContent(text=event.delta, choice_index=event.content_index)], + choice_index=event.content_index, + inner_content=event, + ), True + + def response_audio_transcript_done_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response audio transcript done.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[StreamingTextContent(text=event.transcript, choice_index=event.content_index)], + choice_index=event.content_index, + inner_content=event, + ), False + + def response_function_call_arguments_delta_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> tuple[Any, bool]: + """Handle response function call arguments delta.""" + return StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + ], + choice_index=0, + inner_content=event, + ), True + + def error_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> None: + """Handle error.""" + logger.error("Error received: %s", event.error) + + def session_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> None: + """Handle session.""" + logger.debug("Session created or updated, session: %s", event.session) + + async def response_function_call_arguments_done_callback( + self, + event: RealtimeServerEvent, + settings: PromptExecutionSettings | None = None, + **kwargs: Any, + ) -> None: + """Handle response function call done.""" + item = FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + kernel: Kernel | None = kwargs.get("kernel") + call_id = item.name + function_name = next( + output_item_event.item.name + for output_item_event in self.event_log[ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED] + if output_item_event.item.call_id == call_id + ) + item.plugin_name, item.function_name = function_name.split("-", 1) + if kernel: + chat_history = ChatHistory() + await kernel.invoke_function_call(item, chat_history) + await self.input_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": chat_history.messages[-1]})) # The model doesn't start responding to the tool call automatically, so triggering it here. - await self.send_event(SendEvents.RESPONSE_CREATE) + await self.input_buffer.put(SendEvents.RESPONSE_CREATE) return chat_history.messages[-1], False + # region settings + + @override def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: - """Get the request settings class.""" from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa OpenAIRealtimeExecutionSettings, ) diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index ebdd4eed3739..c9a48f9d45b0 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -1,40 +1,27 @@ # Copyright (c) Microsoft. All rights reserved. +import sys from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, Callable +from asyncio import Queue +from collections.abc import Callable from typing import TYPE_CHECKING, Any, ClassVar +from pydantic import Field + +if sys.version_info >= (3, 11): + from asyncio import TaskGroup +else: + from taskgroup import TaskGroup + from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent from semantic_kernel.services.ai_service_client_base import AIServiceClientBase from semantic_kernel.utils.experimental_decorator import experimental_class if TYPE_CHECKING: from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.contents.chat_history import ChatHistory - from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent - -#### -# TODO (eavanvalkenburg): Move to ADR -# Receiving: -# Option 1: Events and Contents split -# - content received through main receive_content method -# - events received through event callback handlers -# Option 2: Everything is Content -# - content (events as new Content Type) received through main receive_content method -# Option 3: Everything is Event (current) -# - receive_content method is removed -# - events received through main listen method -# - default event handlers added for things like errors and function calling -# - built-in vs custom event handling - separate or not? -# Sending: -# Option 1: Events and Contents split -# - send_content and send_event -# Option 2: Everything is Content -# - single method needed, with EventContent type support -# Option 3: Everything is Event (current) -# - send_event method only, Content is part of event data -#### @experimental_class @@ -42,6 +29,8 @@ class RealtimeClientBase(AIServiceClientBase, ABC): """Base class for a realtime client.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False + input_buffer: Queue[tuple[str, dict[str, Any]] | str] = Field(default_factory=Queue) + output_buffer: Queue[tuple[str, StreamingChatMessageContent]] = Field(default_factory=Queue) async def __aenter__(self) -> "RealtimeClientBase": """Enter the context manager. @@ -94,41 +83,50 @@ async def update_session( """ raise NotImplementedError - @abstractmethod - async def event_listener( + async def start_streaming( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, - ) -> AsyncGenerator["StreamingChatMessageContent", Any]: - """Get text contents from audio. + ) -> None: + """Start streaming, will start both listening and sending. + + This method, start tasks for both listening and sending. + + The arguments are passed to the start_listening method. Args: settings: Prompt execution settings. chat_history: Chat history. kwargs: Additional arguments. - - Yields: - StreamingChatMessageContent messages """ - raise NotImplementedError + async with TaskGroup() as tg: + tg.create_task(self.start_listening(settings=settings, chat_history=chat_history, **kwargs)) + tg.create_task(self.start_sending(**kwargs)) @abstractmethod - async def send_event( + async def start_listening( self, - event: str, - event_data: dict[str, Any] | None = None, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> None: - """Send an event to the session. + """Starts listening for messages from the service, adds them to the output_buffer. Args: - event: Event name, can be a string or a Enum value. - event_data: Event data. + settings: Prompt execution settings. + chat_history: Chat history. kwargs: Additional arguments. """ raise NotImplementedError + @abstractmethod + async def start_sending( + self, + ) -> None: + """Start sending items from the input_buffer to the service.""" + raise NotImplementedError + def _update_function_choice_settings_callback( self, ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: diff --git a/python/semantic_kernel/contents/audio_content.py b/python/semantic_kernel/contents/audio_content.py index 8ee4197aaa8f..77d4a9970a63 100644 --- a/python/semantic_kernel/contents/audio_content.py +++ b/python/semantic_kernel/contents/audio_content.py @@ -3,6 +3,7 @@ import mimetypes from typing import Any, ClassVar, Literal, TypeVar +from numpy import ndarray from pydantic import Field from semantic_kernel.contents.binary_content import BinaryContent @@ -51,3 +52,8 @@ def from_audio_file(cls: type[_T], path: str) -> "AudioContent": def to_dict(self) -> dict[str, Any]: """Convert the instance to a dictionary.""" return {"type": "audio_url", "audio_url": {"uri": str(self)}} + + @classmethod + def from_nd_array(cls: type[_T], data: ndarray, mime_type: str) -> "AudioContent": + """Create an instance from an nd array.""" + return cls(data=data, mime_type=mime_type) diff --git a/python/semantic_kernel/contents/binary_content.py b/python/semantic_kernel/contents/binary_content.py index a36535b0c120..7ac5840dd8fb 100644 --- a/python/semantic_kernel/contents/binary_content.py +++ b/python/semantic_kernel/contents/binary_content.py @@ -5,6 +5,7 @@ from typing import Annotated, Any, ClassVar, Literal, TypeVar from xml.etree.ElementTree import Element # nosec +from numpy import ndarray from pydantic import Field, FilePath, UrlConstraints, computed_field from pydantic_core import Url @@ -48,7 +49,7 @@ def __init__( self, uri: Url | str | None = None, data_uri: DataUrl | str | None = None, - data: str | bytes | None = None, + data: str | bytes | ndarray | None = None, data_format: str | None = None, mime_type: str | None = None, **kwargs: Any, @@ -76,14 +77,17 @@ def __init__( else: kwargs["metadata"] = _data_uri.parameters elif data: - if isinstance(data, str): - _data_uri = DataUri( - data_str=data, data_format=data_format, mime_type=mime_type or self.default_mime_type - ) - else: - _data_uri = DataUri( - data_bytes=data, data_format=data_format, mime_type=mime_type or self.default_mime_type - ) + match data: + case str(): + _data_uri = DataUri( + data_str=data, data_format=data_format, mime_type=mime_type or self.default_mime_type + ) + case bytes(): + _data_uri = DataUri( + data_bytes=data, data_format=data_format, mime_type=mime_type or self.default_mime_type + ) + case ndarray(): + _data_uri = DataUri(data_array=data, mime_type=mime_type or self.default_mime_type) if uri is not None: if isinstance(uri, str) and os.path.exists(uri): @@ -109,8 +113,10 @@ def data_uri(self, value: str): self.metadata.update(self._data_uri.parameters) @property - def data(self) -> bytes: + def data(self) -> bytes | ndarray: """Get the data.""" + if self._data_uri and self._data_uri.data_array: + return self._data_uri.data_array if self._data_uri and self._data_uri.data_bytes: return self._data_uri.data_bytes if self._data_uri and self._data_uri.data_str: @@ -118,15 +124,18 @@ def data(self) -> bytes: return b"" @data.setter - def data(self, value: str | bytes): + def data(self, value: str | bytes | ndarray): """Set the data.""" if self._data_uri: self._data_uri.update_data(value) else: - if isinstance(value, str): - self._data_uri = DataUri(data_str=value, mime_type=self.mime_type) - else: - self._data_uri = DataUri(data_bytes=value, mime_type=self.mime_type) + match value: + case str(): + self._data_uri = DataUri(data_str=value, mime_type=self.mime_type) + case bytes(): + self._data_uri = DataUri(data_bytes=value, mime_type=self.mime_type) + case ndarray(): + self._data_uri = DataUri(data_array=value, mime_type=self.mime_type) @property def mime_type(self) -> str: diff --git a/python/semantic_kernel/contents/utils/data_uri.py b/python/semantic_kernel/contents/utils/data_uri.py index d49022a6b104..d70c175209fa 100644 --- a/python/semantic_kernel/contents/utils/data_uri.py +++ b/python/semantic_kernel/contents/utils/data_uri.py @@ -12,6 +12,7 @@ else: from typing import Self # type: ignore # pragma: no cover +from numpy import ndarray from pydantic import Field, ValidationError, field_validator, model_validator from pydantic_core import Url @@ -28,16 +29,20 @@ class DataUri(KernelBaseModel, validate_assignment=True): data_bytes: bytes | None = None data_str: str | None = None + data_array: ndarray | None = None mime_type: str | None = None parameters: dict[str, str] = Field(default_factory=dict) data_format: str | None = None - def update_data(self, value: str | bytes): + def update_data(self, value: str | bytes | ndarray): """Update the data, using either a string or bytes.""" - if isinstance(value, str): - self.data_str = value - else: - self.data_bytes = value + match value: + case str(): + self.data_str = value + case bytes(): + self.data_bytes = value + case ndarray(): + self.data_array = value @model_validator(mode="before") @classmethod @@ -49,7 +54,13 @@ def _validate_data(cls, values: Any) -> dict[str, Any]: @model_validator(mode="after") def _parse_data(self) -> Self: - """Parse the data bytes to str.""" + """Parse the data bytes to str. + + Will try to decode the data bytes to a string if it is not already set. + However if the data array is used, it will not be converted to a string. + """ + if self.data_array: + return self if not self.data_str and self.data_bytes: if self.data_format and self.data_format.lower() == "base64": self.data_str = base64.b64encode(self.data_bytes).decode("utf-8") @@ -113,10 +124,12 @@ def from_data_uri(cls: type[_T], data_uri: str | Url, default_mime_type: str = " def to_string(self, metadata: dict[str, str] = {}) -> str: """Return the data uri as a string.""" + if self.data_array: + data_str = self.data_array.tobytes().decode("utf-8") parameters = ";".join([f"{key}={val}" for key, val in metadata.items()]) parameters = f";{parameters}" if parameters else "" data_format = f"{self.data_format}" if self.data_format else "" - return f"data:{self.mime_type or ''}{parameters};{data_format},{self.data_str}" + return f"data:{self.mime_type or ''}{parameters};{data_format},{self.data_str or data_str}" def __eq__(self, value: object) -> bool: """Check if the data uri is equal to another.""" diff --git a/python/uv.lock b/python/uv.lock index 161b91c05ab9..fde73e9c4f0a 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -445,7 +445,7 @@ name = "build" version = "1.2.2.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "os_name == 'nt' and sys_platform == 'win32'" }, + { name = "colorama", marker = "(os_name == 'nt' and sys_platform == 'darwin') or (os_name == 'nt' and sys_platform == 'linux') or (os_name == 'nt' and sys_platform == 'win32')" }, { name = "importlib-metadata", marker = "(python_full_version < '3.10.2' and sys_platform == 'darwin') or (python_full_version < '3.10.2' and sys_platform == 'linux') or (python_full_version < '3.10.2' and sys_platform == 'win32')" }, { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pyproject-hooks", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -688,7 +688,7 @@ name = "click" version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "(platform_system == 'Windows' and sys_platform == 'darwin') or (platform_system == 'Windows' and sys_platform == 'linux') or (platform_system == 'Windows' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ @@ -1837,7 +1837,7 @@ name = "ipykernel" version = "6.29.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "sys_platform == 'darwin'" }, + { name = "appnope", marker = "(platform_system == 'Darwin' and sys_platform == 'darwin') or (platform_system == 'Darwin' and sys_platform == 'linux') or (platform_system == 'Darwin' and sys_platform == 'win32')" }, { name = "comm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "debugpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "ipython", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -2744,7 +2744,7 @@ name = "nvidia-cudnn-cu12" version = "9.1.0.70" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741 }, @@ -2755,7 +2755,7 @@ name = "nvidia-cufft-cu12" version = "11.2.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/27/94/3266821f65b92b3138631e9c8e7fe1fb513804ac934485a8d05776e1dd43/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9", size = 211459117 }, @@ -2774,9 +2774,9 @@ name = "nvidia-cusolver-cu12" version = "11.6.1.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/3a/e1/5b9089a4b2a4790dfdea8b3a006052cfecff58139d5a4e34cb1a51df8d6f/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260", size = 127936057 }, @@ -2787,7 +2787,7 @@ name = "nvidia-cusparse-cu12" version = "12.3.1.170" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763 }, @@ -3408,7 +3408,7 @@ name = "portalocker" version = "2.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, + { name = "pywin32", marker = "(platform_system == 'Windows' and sys_platform == 'darwin') or (platform_system == 'Windows' and sys_platform == 'linux') or (platform_system == 'Windows' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ed/d3/c6c64067759e87af98cc668c1cc75171347d0f1577fab7ca3749134e3cd4/portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f", size = 40891 } wheels = [ @@ -4748,6 +4748,7 @@ dependencies = [ { name = "pybars4", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pydantic", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pydantic-settings", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "taskgroup", marker = "(python_full_version < '3.11' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')" }, ] [package.optional-dependencies] @@ -4783,7 +4784,7 @@ hugging-face = [ { name = "transformers", extra = ["torch"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] milvus = [ - { name = "milvus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, + { name = "milvus", marker = "(platform_system != 'Windows' and sys_platform == 'darwin') or (platform_system != 'Windows' and sys_platform == 'linux') or (platform_system != 'Windows' and sys_platform == 'win32')" }, { name = "pymilvus", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] mistralai = [ @@ -4868,7 +4869,7 @@ requires-dist = [ { name = "google-generativeai", marker = "extra == 'google'", specifier = "~=0.7" }, { name = "ipykernel", marker = "extra == 'notebooks'", specifier = "~=6.29" }, { name = "jinja2", specifier = "~=3.1" }, - { name = "milvus", marker = "sys_platform != 'win32' and extra == 'milvus'", specifier = ">=2.3,<2.3.8" }, + { name = "milvus", marker = "platform_system != 'Windows' and extra == 'milvus'", specifier = ">=2.3,<2.3.8" }, { name = "mistralai", marker = "extra == 'mistralai'", specifier = ">=1.2,<2.0" }, { name = "motor", marker = "extra == 'mongo'", specifier = ">=3.3.2,<3.7.0" }, { name = "nest-asyncio", specifier = "~=1.6" }, @@ -4895,6 +4896,7 @@ requires-dist = [ { name = "redis", extras = ["hiredis"], marker = "extra == 'redis'", specifier = "~=5.0" }, { name = "redisvl", marker = "extra == 'redis'", specifier = ">=0.3.6" }, { name = "sentence-transformers", marker = "extra == 'hugging-face'", specifier = ">=2.2,<4.0" }, + { name = "taskgroup", marker = "python_full_version < '3.11'", specifier = ">=0.2.2" }, { name = "torch", marker = "extra == 'hugging-face'", specifier = "==2.5.1" }, { name = "transformers", extras = ["torch"], marker = "extra == 'hugging-face'", specifier = "~=4.28" }, { name = "types-redis", marker = "extra == 'redis'", specifier = "~=4.6.0.20240425" }, @@ -5154,6 +5156,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/40/44/4a5f08c96eb108af5cb50b41f76142f0afa346dfa99d5296fe7202a11854/tabulate-0.9.0-py3-none-any.whl", hash = "sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f", size = 35252 }, ] +[[package]] +name = "taskgroup" +version = "0.2.2" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "exceptiongroup", marker = "(python_full_version < '3.11' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')" }, + { name = "typing-extensions", marker = "(python_full_version < '3.11' and sys_platform == 'darwin') or (python_full_version < '3.11' and sys_platform == 'linux') or (python_full_version < '3.11' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/f0/8d/e218e0160cc1b692e6e0e5ba34e8865dbb171efeb5fc9a704544b3020605/taskgroup-0.2.2.tar.gz", hash = "sha256:078483ac3e78f2e3f973e2edbf6941374fbea81b9c5d0a96f51d297717f4752d", size = 11504 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d1/b1/74babcc824a57904e919f3af16d86c08b524c0691504baf038ef2d7f655c/taskgroup-0.2.2-py2.py3-none-any.whl", hash = "sha256:e2c53121609f4ae97303e9ea1524304b4de6faf9eb2c9280c7f87976479a52fb", size = 14237 }, +] + [[package]] name = "tenacity" version = "9.0.0" @@ -5257,21 +5272,21 @@ dependencies = [ { name = "fsspec", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "jinja2", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "networkx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, - { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cublas-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-cupti-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cuda-runtime-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cudnn-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cufft-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-curand-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cusolver-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cusparse-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-nccl-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-nvtx-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, { name = "setuptools", marker = "(python_full_version >= '3.12' and sys_platform == 'darwin') or (python_full_version >= '3.12' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, { name = "sympy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "triton", marker = "(python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ @@ -5313,7 +5328,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "sys_platform == 'win32'" }, + { name = "colorama", marker = "(platform_system == 'Windows' and sys_platform == 'darwin') or (platform_system == 'Windows' and sys_platform == 'linux') or (platform_system == 'Windows' and sys_platform == 'win32')" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } wheels = [ @@ -5361,7 +5376,7 @@ name = "triton" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, + { name = "filelock", marker = "(python_full_version < '3.13' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform == 'win32')" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/98/29/69aa56dc0b2eb2602b553881e34243475ea2afd9699be042316842788ff5/triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8", size = 209460013 }, From 7754aabf7c845f3eafb54bed7ac94c5d5c04e79f Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 16 Jan 2025 10:07:51 +0100 Subject: [PATCH 08/25] updated ADR --- docs/decisions/00XX-realtime-api-clients.md | 122 +++++++++++++------- 1 file changed, 81 insertions(+), 41 deletions(-) diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index 1b0bbd2d6c52..96570b389de1 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -79,82 +79,122 @@ Server side events: ## Decision Drivers - - Simple programming model that is likely able to handle future realtime api's and evolution of the existing ones. -- Support for the most common scenario's and content, extensible for the rest. -- Natively integrated with Semantic Kernel especially for content types and function calling. -- Support multiple types of connections, like websocket and WebRTC - +- Whenever possible we transform incoming content into Semantic Kernel content, but surface everything, so it's extensible +- Protocol agnostic, should be able to use different types of protocols under the covers, like websocket and WebRTC, without changing the client code (unless the protocol requires it). + ## Decision driver questions - For WebRTC, a audio device can be passed, should this be a requirement for the client also for websockets? -## Considered Options +There are multiple areas where we need to make decisions, these are: +- Content and Events +- Programming model +- Audio speaker/microphone handling +# Content and Events + +## Considered Options - Content and Events Both the sending and receiving side of these integrations need to decide how to deal with the api's. -- Treat content events separate from control events -- Treat everything as content items -- Treat everything as events +1. Treat content events separate from control events +1. Treat everything as content items +1. Treat everything as events -### Treat content events separate from control events +### 1. Treat content events separate from control events This would mean there are two mechanisms in the clients, one deals with content, and one with control events. - Pro: - strongly typed responses for known content - easy to use as the main interactions are clear with familiar SK content types, the rest goes through a separate mechanism - - this might fit better with something like WebRTC that has distinct channels for audio and video vs a data stream for all other events - Con: - new content support requires updates in the codebase and can be considered breaking (potentially sending additional types back) - additional complexity in dealing with two streams of data -### Treat everything as content items +### 2. Treat everything as content items +This would mean that all events are turned into Semantic Kernel content items, and would also mean that we need to define additional content types for the control events. +- Pro: + - everything is a content item, so it's easy to deal with +- Con: + - overkill for simple control events -## Decision Outcome +### 3. Treat everything as events +This would mean that all events are retained and returned to the developer as is, without any transformation. -Chosen option: "{title of option 1}", because -{justification. e.g., only option, which meets k.o. criterion decision driver | which resolves force {force} | … | comes out best (see below)}. +- Pro: + - no transformation needed + - easy to maintain +- Con: + - nothing easing the burden on the developer, they need to deal with the raw events + - no way to easily switch between one provider and another - +## Decision Outcome - Content and Events -### Consequences +Chosen option: ... -- Good, because {positive consequence, e.g., improvement of one or more desired qualities, …} -- Bad, because {negative consequence, e.g., compromising one or more desired qualities, …} -- … +# Programming model - +## Considered Options - Programming model +The programming model for the clients needs to be simple and easy to use, while also being able to handle the complexity of the realtime api's. -## Validation +_In this section we will refer to events for both content and events, regardless of the decision made in the previous section._ -{describe how the implementation of/compliance with the ADR is validated. E.g., by a review or an ArchUnit test} +1. Async generator for receiving events, that yields contents, combined with a event handler/callback mechanism for receiving events and a function for sending events + - 1a: Single event handlers, where each event is passed to the handler + - 1b: Multiple event handlers, where each event type has its own handler +2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers - +### 1. Async generator for receiving events, that yields contents, combined with a event handler/callback mechanism for receiving events and a function for sending events +This would mean that the client would have a mechanism to register event handlers, and the integration would call these handlers when an event is received. For sending events, a function would be created that sends the event to the service. -## Pros and Cons of the Options +- Pro: + - without any additional setup you get content back, just as with "regular" chat models + - event handlers are mostly for more complex interactions, so ok to be slightly more complex +- Con: + - developer judgement needs to be made (or exposed with parameters) on what is returned through the async generator and what is passed to the event handlers -### {title of option 1} +### 2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers +This would mean that the there are two queues, one for sending and one for receiving, and the developer can listen to the receiving queue and send to the sending queue. Internal things like auto-function calling can listen in on the same queue and act on it, and put a message back on the sending queue with ease. - +- Pro: + - simple to use, just start sending and start receiving + - easy to understand, as queues are a well known concept + - developers can just skip events they are not interested in +- Con: + - potentially causes audio delays because of the queueing mechanism + +## Decision Outcome - Programming model + +Chosen option: ... + +# Audio speaker/microphone handling + +## Considered Options - Audio speaker/microphone handling -{example | description | pointer to more information | …} +1. Create abstraction in SK for audio handlers, that can be passed into the realtime client to record and play audio +2. Send and receive AudioContent (wrapped in StreamingChatMessageContent) to the client, and let the client handle the audio recording and playing -- Good, because {argument a} -- Good, because {argument b} - -- Neutral, because {argument c} -- Bad, because {argument d} -- … +### 1. Create abstraction in SK for audio handlers, that can be passed into the realtime client to record and play audio +This would mean that the client would have a mechanism to register audio handlers, and the integration would call these handlers when audio is received or needs to be sent. A additional abstraction for this would have to be created in Semantic Kernel (or potentially taken from a standard). -### {title of other option} +- Pro: + - simple/local audio handlers can be shipped with SK making it easy to use + - extensible by third parties to integrate into other systems (like Azure Communications Service) + - could mitigate buffer issues by prioritizing audio content being sent to the handlers +- Con: + - extra code in SK that needs to be maintained, potentially relying on third party code + +### 2. Send and receive AudioContent (wrapped in StreamingChatMessageContent) to the client, and let the client handle the audio recording and playing +This would mean that the client would receive AudioContent items, and would have to deal with them itself, including recording and playing the audio. + +- Pro: + - no extra code in SK that needs to be maintained +- Con: + - extra burden on the developer to deal with the audio -{example | description | pointer to more information | …} +## Decision Outcome - Audio speaker/microphone handling -- Good, because {argument a} -- Good, because {argument b} -- Neutral, because {argument c} -- Bad, because {argument d} -- … +Chosen option: ... From 284475bab5117ed89a9262d1fea5035332711c51 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 14:41:29 +0100 Subject: [PATCH 09/25] webrtc working! --- docs/decisions/00XX-realtime-api-clients.md | 4 +- .../audio/04-chat_with_realtime_api.py | 167 +++---- .../concepts/audio/audio_player_async.py | 75 --- .../concepts/audio/audio_recorder_stream.py | 77 --- .../open_ai/services/open_ai_config_base.py | 5 +- .../ai/open_ai/services/open_ai_realtime.py | 10 +- .../open_ai/services/open_ai_realtime_base.py | 455 +++++++++--------- .../ai/open_ai/settings/open_ai_settings.py | 4 + .../connectors/ai/realtime_client_base.py | 5 +- .../connectors/ai/realtime_helpers.py | 190 ++++++++ .../semantic_kernel/contents/audio_content.py | 4 +- .../contents/binary_content.py | 20 +- .../contents/utils/data_uri.py | 11 +- python/uv.lock | 54 +-- 14 files changed, 539 insertions(+), 542 deletions(-) delete mode 100644 python/samples/concepts/audio/audio_player_async.py delete mode 100644 python/samples/concepts/audio/audio_recorder_stream.py create mode 100644 python/semantic_kernel/connectors/ai/realtime_helpers.py diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index 96570b389de1..6fcf0972aea2 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -12,7 +12,7 @@ informed: ## Context and Problem Statement -Multiple model providers are starting to enable realtime voice-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. The key addition that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. +Multiple model providers are starting to enable realtime voice-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. The key addition that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. There are also options for Google to use video and images as input, so really it is multimodal, but for now we are focusing on the voice-to-voice part, while keeping in mind that video is coming. The way these API's work at this time is through either Websockets or WebRTC. @@ -154,7 +154,7 @@ This would mean that the client would have a mechanism to register event handler - developer judgement needs to be made (or exposed with parameters) on what is returned through the async generator and what is passed to the event handlers ### 2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers -This would mean that the there are two queues, one for sending and one for receiving, and the developer can listen to the receiving queue and send to the sending queue. Internal things like auto-function calling can listen in on the same queue and act on it, and put a message back on the sending queue with ease. +This would mean that the there are two queues, one for sending and one for receiving, and the developer can listen to the receiving queue and send to the sending queue. Internal things like parsing events to content types and auto-function calling are processed first, and the result is put in the queue, the content type should use inner_content to capture the full event and these might add a message to the send queue as well. - Pro: - simple to use, just start sending and start receiving diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index 40f16a2c0a24..0f895f7dc9dc 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -1,17 +1,11 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio -import contextlib import logging import signal -from typing import Any +from random import randint -import numpy as np -from aiortc.mediastreams import MediaStreamError, MediaStreamTrack -from av import AudioFrame -from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent +import sounddevice as sd -from samples.concepts.audio.audio_player_async import AudioPlayerAsync -from samples.concepts.audio.audio_recorder_stream import AudioRecorderStream from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai import ( @@ -19,12 +13,18 @@ OpenAIRealtimeWebRTC, TurnDetection, ) -from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_base import ListenEvents from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.contents import AudioContent, ChatHistory, StreamingTextContent +from semantic_kernel.connectors.ai.realtime_helpers import SKSimplePlayer +from semantic_kernel.contents import ChatHistory +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent from semantic_kernel.functions import kernel_function logging.basicConfig(level=logging.WARNING) +aiortc_log = logging.getLogger("aiortc") +aiortc_log.setLevel(logging.WARNING) +aioice_log = logging.getLogger("aioice") +aioice_log.setLevel(logging.WARNING) logger = logging.getLogger(__name__) # This simple sample demonstrates how to use the OpenAI Realtime API to create @@ -34,7 +34,8 @@ # - pyaudio # - sounddevice # - pydub -# e.g. pip install semantic-kernel[openai_realtime] pyaudio sounddevice pydub +# - aiortc +# e.g. pip install pyaudio sounddevice pydub # The characterics of your speaker and microphone are a big factor in a smooth conversation # so you may need to try out different devices for each. @@ -45,124 +46,66 @@ def check_audio_devices(): - import sounddevice as sd # type: ignore - - print(sd.query_devices()) + logger.info(sd.query_devices()) check_audio_devices() -class Speaker: - """This is a simple class that opens the session with the realtime api and plays the audio response. +class ReceivingStreamHandler: + """This is a simple class that listens to the received buffer of the RealtimeClientBase. + + It can be used to play audio and print the transcript of the conversation. - At the same time it prints the transcript of the conversation to the console. + It can also be used to act on other events from the service. """ - def __init__(self, audio_player: AudioPlayerAsync, realtime_client: RealtimeClientBase, kernel: Kernel): + def __init__(self, realtime_client: RealtimeClientBase, audio_player: SKSimplePlayer | None = None): self.audio_player = audio_player self.realtime_client = realtime_client - self.kernel = kernel - async def play( + async def listen( self, - chat_history: ChatHistory, - settings: OpenAIRealtimeExecutionSettings, + play_audio: bool = True, print_transcript: bool = True, ) -> None: - # reset the frame count for the audio player - self.audio_player.reset_frame_count() # print the start message of the transcript if print_transcript: print("Mosscap (transcript): ", end="") try: # start listening for events while True: - _, content = await self.realtime_client.output_buffer.get() - if not content: - continue - # the contents returned should be StreamingChatMessageContent - # so we will loop through the items within it. - for item in content.items: - match item: - case StreamingTextContent(): - if print_transcript: - print(item.text, end="") - await asyncio.sleep(0.01) - continue - case AudioContent(): - self.audio_player.add_data(item.data) - await asyncio.sleep(0.01) - continue + event_type, event = await self.realtime_client.receive_buffer.get() + match event_type: + case ListenEvents.RESPONSE_AUDIO_DELTA: + if play_audio and self.audio_player and isinstance(event, StreamingChatMessageContent): + await self.audio_player.add_audio(event.items[0]) + case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA: + if print_transcript and isinstance(event, StreamingChatMessageContent): + print(event.content, end="") + case ListenEvents.RESPONSE_CREATED: + if print_transcript: + print("") + await asyncio.sleep(0.01) except asyncio.CancelledError: print("\nThanks for talking to Mosscap!") -class Microphone(MediaStreamTrack): - """This is a simple class that opens the microphone and sends the audio to the realtime api.""" - - kind = "audio" - - def __init__(self, audio_recorder: AudioRecorderStream, realtime_client: RealtimeClientBase): - self.audio_recorder = audio_recorder - self.realtime_client = realtime_client - self.queue = asyncio.Queue() - self.loop = asyncio.get_running_loop() - self._pts = 0 - - async def recv(self) -> Any: - # start the audio recording - try: - return await self.queue.get() - except Exception as e: - logger.error(f"Error receiving audio frame: {str(e)}") - raise MediaStreamError("Failed to receive audio frame") - - async def record_audio(self): - def callback(indata, frames, time, status): - if status: - logger.warning(f"Audio input status: {status}") - audio_data = indata.copy() - - if audio_data.dtype != np.int16: - audio_data = (audio_data * 32767).astype(np.int16) - - # Create AudioFrame with incrementing pts - frame = AudioFrame( - samples=len(audio_data), - layout="mono", - format="s16", # 16-bit signed integer - ) - frame.rate = 48000 - frame.pts = self._pts - self._pts += len(audio_data) # Increment pts by frame size - - frame.planes[0].update(audio_data.tobytes()) - - asyncio.run_coroutine_threadsafe(self.queue.put(frame), self.loop) - - await self.realtime_client.input_buffer.put("response.create") - await self.audio_recorder.stream_audio_content_with_callback(callback=callback) - - # this function is used to stop the processes when ctrl + c is pressed def signal_handler(): for task in asyncio.all_tasks(): task.cancel() +weather_conditions = ["sunny", "hot", "cloudy", "raining", "freezing", "snowing"] + + @kernel_function def get_weather(location: str) -> str: """Get the weather for a location.""" - logger.debug(f"Getting weather for {location}") - return f"The weather in {location} is sunny." - - -def response_created_callback( - event: RealtimeServerEvent, settings: PromptExecutionSettings | None = None, **kwargs: Any -) -> None: - """Add a empty print to start a new line for a new response.""" - print("") + weather = weather_conditions[randint(0, len(weather_conditions))] # nosec + logger.warning(f"Getting weather for {location}: {weather}") + return f"The weather in {location} is {weather}." async def main() -> None: @@ -174,20 +117,20 @@ async def main() -> None: kernel = Kernel() kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) - # create the realtime client and register the response created callback - realtime_client = OpenAIRealtimeWebRTC(ai_model_id="gpt-4o-realtime-preview-2024-12-17") - realtime_client.register_event_handler("response.created", response_created_callback) + # create the realtime client and optionally add the audio output function, this is optional + audio_player = SKSimplePlayer() + realtime_client = OpenAIRealtimeWebRTC(audio_output=audio_player.realtime_client_callback) - # create the speaker and microphone - speaker = Speaker(AudioPlayerAsync(device_id=None), realtime_client, kernel) - microphone = Microphone(AudioRecorderStream(device_id=None), realtime_client) + # create stream receiver, this can play the audio, if the audio_player is passed + # and allows you to print the transcript of the conversation + # and review or act on other events from the service + stream_handler = ReceivingStreamHandler(realtime_client) # SimplePlayer(device_id=None) # Create the settings for the session # the key thing to decide on is to enable the server_vad turn detection # if turn is turned off (by setting turn_detection=None), you will have to send # the "input_audio_buffer.commit" and "response.create" event to the realtime api # to signal the end of the user's turn and start the response. - # The realtime api, does not use a system message, but takes instructions as a parameter for a session instructions = """ You are a chat bot. Your name is Mosscap and @@ -197,7 +140,7 @@ async def main() -> None: effectively, but you tend to answer with long flowery prose. """ - # but we can add a chat history to conversation after starting it + # and we can add a chat history to conversation after starting it chat_history = ChatHistory() chat_history.add_user_message("Hi there, who are you?") chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") @@ -208,14 +151,14 @@ async def main() -> None: turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), function_choice_behavior=FunctionChoiceBehavior.Auto(), ) - async with realtime_client: - await realtime_client.update_session(settings=settings, chat_history=chat_history) - await realtime_client.start_listening(settings, chat_history) - await realtime_client.start_sending(input_audio_track=microphone) - # await realtime_client.start_streaming(settings, chat_history, input_audio_track=microphone) - # start the the speaker and the microphone - with contextlib.suppress(asyncio.CancelledError): - await speaker.play(chat_history, settings) + # the context manager calls the create_session method on the client and start listening to the audio stream + async with realtime_client, audio_player: + await realtime_client.update_session( + settings=settings, chat_history=chat_history, kernel=kernel, create_response=True + ) + async with asyncio.TaskGroup() as tg: + tg.create_task(realtime_client.start_streaming()) + tg.create_task(stream_handler.listen()) if __name__ == "__main__": diff --git a/python/samples/concepts/audio/audio_player_async.py b/python/samples/concepts/audio/audio_player_async.py deleted file mode 100644 index 36c1492094a6..000000000000 --- a/python/samples/concepts/audio/audio_player_async.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -import threading - -import numpy as np -import pyaudio -import sounddevice as sd - -CHUNK_LENGTH_S = 0.05 # 100ms -SAMPLE_RATE = 24000 -FORMAT = pyaudio.paInt16 -CHANNELS = 1 - - -class AudioPlayerAsync: - def __init__(self, device_id: int | None = None): - self.queue = [] - self.lock = threading.Lock() - self.stream = sd.OutputStream( - callback=self.callback, - samplerate=SAMPLE_RATE, - channels=CHANNELS, - dtype=np.int16, - blocksize=int(CHUNK_LENGTH_S * SAMPLE_RATE), - device=device_id, - ) - self.playing = False - self._frame_count = 0 - - def callback(self, outdata, frames, time, status): # noqa - with self.lock: - data = np.empty(0, dtype=np.int16) - - # get next item from queue if there is still space in the buffer - while len(data) < frames and len(self.queue) > 0: - item = self.queue.pop(0) - frames_needed = frames - len(data) - data = np.concatenate((data, item[:frames_needed])) - if len(item) > frames_needed: - self.queue.insert(0, item[frames_needed:]) - - self._frame_count += len(data) - - # fill the rest of the frames with zeros if there is no more data - if len(data) < frames: - data = np.concatenate((data, np.zeros(frames - len(data), dtype=np.int16))) - - outdata[:] = data.reshape(-1, 1) - - def reset_frame_count(self): - self._frame_count = 0 - - def get_frame_count(self): - return self._frame_count - - def add_data(self, data: bytes | np.ndarray): - with self.lock: - # bytes is pcm16 single channel audio data, convert to numpy array - np_data = np.frombuffer(data, dtype=np.int16) if isinstance(data, bytes) else data - self.queue.append(np_data) - if not self.playing: - self.start() - - def start(self): - self.playing = True - self.stream.start() - - def stop(self): - self.playing = False - self.stream.stop() - with self.lock: - self.queue = [] - - def terminate(self): - self.stream.close() diff --git a/python/samples/concepts/audio/audio_recorder_stream.py b/python/samples/concepts/audio/audio_recorder_stream.py deleted file mode 100644 index 20c758af3e39..000000000000 --- a/python/samples/concepts/audio/audio_recorder_stream.py +++ /dev/null @@ -1,77 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -import asyncio -import base64 -from collections.abc import AsyncGenerator, Callable -from typing import Any, ClassVar, cast - -import sounddevice as sd -from pydantic import BaseModel - -from semantic_kernel.contents.audio_content import AudioContent - - -class AudioRecorderStream(BaseModel): - """A class to record audio from the microphone and save it to a WAV file. - - To start recording, press the spacebar. To stop recording, release the spacebar. - - To use as a context manager, that automatically removes the output file after exiting the context: - ``` - with AudioRecorder(output_filepath="output.wav") as recorder: - recorder.start_recording() - # Do something with the recorded audio - ... - ``` - """ - - # Audio recording parameters - CHANNELS: ClassVar[int] = 1 - SAMPLE_RATE: ClassVar[int] = 24000 - CHUNK_LENGTH_S: ClassVar[float] = 0.05 - device_id: int | None = None - - async def stream_audio_content_with_callback(self, callback: Callable[..., Any]) -> None: - stream = sd.InputStream( - channels=self.CHANNELS, - samplerate=self.SAMPLE_RATE, - dtype="int16", - device=self.device_id, - callback=callback, - ) - stream.start() - try: - while True: - await asyncio.sleep(0) - except KeyboardInterrupt: - pass - finally: - stream.stop() - stream.close() - - async def stream_audio_content(self) -> AsyncGenerator[AudioContent, None]: - # device_info = sd.query_devices() - # print(device_info) - - read_size = int(self.SAMPLE_RATE * 0.02) - - stream = sd.InputStream( - channels=self.CHANNELS, - samplerate=self.SAMPLE_RATE, - dtype="int16", - device=self.device_id, - ) - stream.start() - try: - while True: - if stream.read_available < read_size: - await asyncio.sleep(0) - continue - - data, _ = stream.read(read_size) - yield AudioContent(data=base64.b64encode(cast(Any, data)), data_format="base64", mime_type="audio/wav") - except KeyboardInterrupt: - pass - finally: - stream.stop() - stream.close() diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py index d3d72795665b..7883be04f4ff 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_config_base.py @@ -3,6 +3,7 @@ import logging from collections.abc import Mapping from copy import copy +from typing import Any from openai import AsyncOpenAI from pydantic import ConfigDict, Field, validate_call @@ -30,6 +31,7 @@ def __init__( default_headers: Mapping[str, str] | None = None, client: AsyncOpenAI | None = None, instruction_role: str | None = None, + **kwargs: Any, ) -> None: """Initialize a client for OpenAI services. @@ -51,6 +53,7 @@ def __init__( client (AsyncOpenAI): An existing OpenAI client, optional. instruction_role (str): The role to use for 'instruction' messages, for example, summarization prompts could use `developer` or `system`. (Optional) + kwargs: Additional keyword arguments. """ # Merge APP_INFO into the headers if it exists @@ -76,7 +79,7 @@ def __init__( args["service_id"] = service_id if instruction_role: args["instruction_role"] = instruction_role - super().__init__(**args) + super().__init__(**args, **kwargs) def to_dict(self) -> dict[str, str]: """Create a dict of the service settings.""" diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index 39c85816ced3..412d0814feb8 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from collections.abc import Mapping +from typing import Any from openai import AsyncOpenAI from pydantic import ValidationError @@ -82,6 +83,7 @@ def __init__( async_client: AsyncOpenAI | None = None, env_file_path: str | None = None, env_file_encoding: str | None = None, + **kwargs: Any, ) -> None: """Initialize an OpenAITextCompletion service. @@ -99,25 +101,27 @@ def __init__( env_file_path (str | None): Use the environment settings file as a fallback to environment variables. (Optional) env_file_encoding (str | None): The encoding of the environment settings file. (Optional) + kwargs: Additional arguments. """ try: openai_settings = OpenAISettings.create( api_key=api_key, org_id=org_id, - text_model_id=ai_model_id, + realtime_model_id=ai_model_id, env_file_path=env_file_path, env_file_encoding=env_file_encoding, ) except ValidationError as ex: raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex - if not openai_settings.text_model_id: + if not openai_settings.realtime_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") super().__init__( - ai_model_id=openai_settings.text_model_id, + ai_model_id=openai_settings.realtime_model_id, service_id=service_id, api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None, org_id=openai_settings.org_id, ai_model_type=OpenAIModelTypes.TEXT, default_headers=default_headers, client=async_client, + **kwargs, ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py index f82bce19164f..e387ef4005aa 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -2,13 +2,14 @@ import asyncio import base64 +import contextlib import json import logging import sys -from collections.abc import AsyncGenerator +from collections.abc import AsyncGenerator, Callable, Coroutine from enum import Enum from inspect import isawaitable -from typing import Any, ClassVar, Protocol, runtime_checkable +from typing import Any, ClassVar, Protocol, cast, runtime_checkable if sys.version_info >= (3, 12): from typing import override # pragma: no cover @@ -24,15 +25,25 @@ RTCPeerConnection, RTCSessionDescription, ) +from av import AudioFrame +from openai._models import construct_type_unchecked from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection from openai.types.beta.realtime.conversation_item_create_event_param import ConversationItemParam from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent -from pydantic import Field +from pydantic import Field, PrivateAttr -from semantic_kernel.connectors.ai.function_calling_utils import prepare_settings_for_function_calling +from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration +from semantic_kernel.connectors.ai.function_calling_utils import ( + prepare_settings_for_function_calling, +) +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_utils import ( + update_settings_from_function_call_configuration, +) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase +from semantic_kernel.connectors.ai.realtime_helpers import SKAudioTrack from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.chat_history import ChatHistory from semantic_kernel.contents.function_call_content import FunctionCallContent @@ -46,6 +57,8 @@ logger: logging.Logger = logging.getLogger(__name__) +# region Protocols + @runtime_checkable @experimental_class @@ -77,6 +90,9 @@ def __call__( ... +# region Events + + @experimental_class class SendEvents(str, Enum): """Events that can be sent.""" @@ -126,6 +142,9 @@ class ListenEvents(str, Enum): RATE_LIMITS_UPDATED = "rate_limits.updated" +# region Websocket + + @experimental_class class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): """OpenAI Realtime service.""" @@ -437,8 +456,6 @@ async def response_function_call_arguments_done_callback( await self.start_sending(SendEvents.RESPONSE_CREATE) return chat_history.messages[-1], False - # region settings - @override def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa @@ -448,6 +465,9 @@ def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"] return OpenAIRealtimeExecutionSettings +# region WebRTC + + @experimental_class class OpenAIRealtimeWebRTCBase(OpenAIHandler, RealtimeClientBase): """OpenAI WebRTC Realtime service.""" @@ -455,135 +475,127 @@ class OpenAIRealtimeWebRTCBase(OpenAIHandler, RealtimeClientBase): SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True peer_connection: RTCPeerConnection | None = None data_channel: RTCDataChannel | None = None - connection: AsyncRealtimeConnection | None = None - connected: asyncio.Event = Field(default_factory=asyncio.Event) - event_log: dict[str, list[RealtimeServerEvent]] = Field(default_factory=dict) - event_handlers: dict[str, list[EventCallBackProtocol | EventCallBackProtocolAsync]] = Field(default_factory=dict) + audio_output: Callable[[AudioFrame], Coroutine[Any, Any, None] | None] | None = None + kernel: Kernel | None = None - def model_post_init(self, *args, **kwargs) -> None: - """Post init method for the model.""" - # Register the default event handlers - self.register_event_handler( - ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA, self.response_audio_transcript_delta_callback - ) - self.register_event_handler( - ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE, self.response_audio_transcript_done_callback - ) - self.register_event_handler( - ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, self.response_function_call_arguments_delta_callback - ) - self.register_event_handler(ListenEvents.ERROR, self.error_callback) - self.register_event_handler(ListenEvents.SESSION_CREATED, self.session_callback) - self.register_event_handler(ListenEvents.SESSION_UPDATED, self.session_callback) - - def register_event_handler( - self, event_type: str | ListenEvents, handler: EventCallBackProtocol | EventCallBackProtocolAsync - ) -> None: - """Register a event handler.""" - if not isinstance(event_type, ListenEvents): - event_type = ListenEvents(event_type) - self.event_handlers.setdefault(event_type, []).append(handler) + _current_settings: PromptExecutionSettings | None = PrivateAttr(None) + _call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict) @override async def start_listening( self, - settings: "PromptExecutionSettings", + settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, - ) -> AsyncGenerator[StreamingChatMessageContent, Any]: - ice_servers = [RTCIceServer(urls=["stun:stun.l.google.com:19302"])] - self.peer_connection = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) + ) -> None: + pass - @self.peer_connection.on("track") - async def on_track(track: MediaStreamTrack) -> None: - if track.kind == "audio": - while True: - frame = await track.recv() - await self.output_buffer.put( - ( - ListenEvents.RESPONSE_AUDIO_DELTA, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[AudioContent(data=frame.to_ndarray(), data_format="base64")], - choice_index=0, - inner_content=frame, - ), + async def _on_track(self, track: MediaStreamTrack) -> None: + logger.info(f"Received {track.kind} track from remote") + if track.kind != "audio": + return + while True: + try: + # This is a MediaStreamTrack, so the type is AudioFrame + # this might need to be updated if video becomes part of this + frame: AudioFrame = await track.recv() # type: ignore + except Exception as e: + logger.error(f"Error getting audio frame: {e!s}") + break + + try: + if self.audio_output: + out = self.audio_output(frame) + if isawaitable(out): + await out + + except Exception as e: + logger.error(f"Error playing remote audio frame: {e!s}") + try: + await self.receive_buffer.put( + ( + ListenEvents.RESPONSE_AUDIO_DELTA, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame)], # type: ignore + choice_index=0, ), - ) - - data_channel = self.peer_connection.createDataChannel("oai-events") - - @data_channel.on("message") - async def on_data(data: bytes) -> None: - event = RealtimeServerEvent.model_validate_strings(data) - event_type = ListenEvents(event.type) - self.event_log.setdefault(event_type, []).append(event) - for handler in self.event_handlers.get(event_type, []): - task = handler(event=event, settings=settings) - if not task: - continue - if isawaitable(task): - async_result = await task - if not async_result: - continue - result, should_return = async_result - else: - result, should_return = task - if should_return: - yield result - else: - chat_history.add_message(result) + ), + ) + except Exception as e: + logger.error(f"Error processing remote audio frame: {e!s}") + await asyncio.sleep(0.01) - offer = await self.peer_connection.createOffer() - await self.peer_connection.setLocalDescription(offer) + async def _on_data(self, data: str) -> None: + """This method is called whenever a data channel message is received. + The data is parsed into a RealtimeServerEvent (by OpenAI) and then processed. + """ try: - ephemeral_token = await self.get_ephemeral_token() - headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"} - - async with ( - ClientSession() as session, - session.post( - f"{self.client.beta.realtime._client.base_url}/realtime/sessions?model={self.ai_model_id}", - headers=headers, - data=offer.sdp, - ) as response, - ): - if response.status not in [200, 201]: - error_text = await response.text() - raise Exception(f"OpenAI WebRTC error: {error_text}") - - sdp_answer = await response.text() - answer = RTCSessionDescription(sdp=sdp_answer, type="answer") - await self.peer_connection.setRemoteDescription(answer) - + event = cast( + RealtimeServerEvent, + construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)), + ) except Exception as e: - logger.error(f"Failed to connect to OpenAI: {e!s}") - raise + logger.error(f"Failed to parse event {data} with error: {e!s}") + return + match event.type: + case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA: + await self.receive_buffer.put(( + event.type, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + content=event.delta, + choice_index=event.content_index, + inner_content=event, + ), + )) + case ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED: + if event.item.type == "function_call": + self._call_id_to_function_map[event.item.call_id] = event.item.name + case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA: + await self.receive_buffer.put(( + event.type, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + ], + choice_index=0, + inner_content=event, + ), + )) + case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE: + await self._handle_function_call_arguments_done(event) + case ListenEvents.ERROR: + logger.error("Error received: %s", event.error) + case ListenEvents.SESSION_CREATED, ListenEvents.SESSION_UPDATED: + logger.info("Session created or updated, session: %s", event.session) + case _: + logger.debug(f"Received event: {event}") + # we put all event in the output buffer, but after the interpreted one. + await self.receive_buffer.put((event.type, event)) @override - async def start_sending(self, input_audio_track: MediaStreamTrack | None = None, **kwargs: Any) -> None: - if input_audio_track: - if not self.peer_connection: - raise ValueError("Peer connection is not established.") - self.peer_connection.addTransceiver(input_audio_track) - - if not self.data_channel: - raise ValueError("Data channel is not established.") + async def start_sending(self, **kwargs: Any) -> None: while True: - item = await self.input_buffer.get() + item = await self.send_buffer.get() if not item: continue if isinstance(item, tuple): event, data = item else: event = item - data = None + data = {} if not isinstance(event, SendEvents): event = SendEvents(event) - response: dict[str, Any] = { - "type": event, - } + response: dict[str, Any] = {"type": event.value} match event: case SendEvents.SESSION_UPDATE: if "settings" not in data: @@ -651,170 +663,153 @@ async def start_sending(self, input_audio_track: MediaStreamTrack | None = None, if "response_id" in data: response["response_id"] = data["response_id"] - self.data_channel.send(json.dumps(response)) + if self.data_channel: + while self.data_channel.readyState != "open": + await asyncio.sleep(0.1) + try: + self.data_channel.send(json.dumps(response)) + except Exception as e: + logger.error(f"Failed to send event {event} with error: {e!s}") @override async def create_session( self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, + audio_track: MediaStreamTrack | None = None, **kwargs: Any, ) -> None: """Create a session in the service.""" - if settings or chat_history or kwargs: - await self.update_session(settings=settings, chat_history=chat_history, **kwargs) + ice_servers = [RTCIceServer(urls=["stun:stun.l.google.com:19302"])] + self.peer_connection = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) - async def get_ephemeral_token(self) -> str: - """Get an ephemeral token from OpenAI.""" - headers = {"Authorization": f"Bearer {self.client.api_key}", "Content-Type": "application/json"} - data = {"model": self.ai_model_id, "voice": "echo"} + self.peer_connection.on("track")(self._on_track) + + self.data_channel = self.peer_connection.createDataChannel("oai-events", protocol="json") + self.data_channel.on("message")(self._on_data) + + self.peer_connection.addTransceiver(audio_track or SKAudioTrack(), "sendrecv") + + offer = await self.peer_connection.createOffer() + await self.peer_connection.setLocalDescription(offer) try: + ephemeral_token = await self.get_ephemeral_token() + headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"} + async with ( ClientSession() as session, session.post( - f"{self.client.beta.realtime._client.base_url}/realtime/sessions", headers=headers, json=data + f"{self.client.beta.realtime._client.base_url}realtime?model={self.ai_model_id}", + headers=headers, + data=offer.sdp, ) as response, ): if response.status not in [200, 201]: error_text = await response.text() - raise Exception(f"Failed to get ephemeral token: {error_text}") + raise Exception(f"OpenAI WebRTC error: {error_text}") - result = await response.json() - return result["client_secret"]["value"] + sdp_answer = await response.text() + answer = RTCSessionDescription(sdp=sdp_answer, type="answer") + await self.peer_connection.setRemoteDescription(answer) + logger.info("Connected to OpenAI WebRTC") except Exception as e: - logger.error(f"Failed to get ephemeral token: {e!s}") + logger.error(f"Failed to connect to OpenAI: {e!s}") raise + if settings or chat_history or kwargs: + await self.update_session(settings=settings, chat_history=chat_history, **kwargs) + @override async def update_session( - self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any + self, + settings: PromptExecutionSettings | None = None, + chat_history: ChatHistory | None = None, + create_response: bool = True, + **kwargs: Any, ) -> None: + if "kernel" in kwargs: + self.kernel = kwargs["kernel"] if settings: - if "kernel" in kwargs: - settings = prepare_settings_for_function_calling( - settings, - self.get_prompt_execution_settings_class(), - self._update_function_choice_settings_callback(), - kernel=kwargs.get("kernel"), # type: ignore - ) - await self.input_buffer.put((SendEvents.SESSION_UPDATE, {"settings": settings})) + self._current_settings = settings + if self._current_settings and self.kernel: + self._current_settings = prepare_settings_for_function_calling( + self._current_settings, + self.get_prompt_execution_settings_class(), + self._update_function_choice_settings_callback(), + kernel=self.kernel, # type: ignore + ) + await self.send_buffer.put((SendEvents.SESSION_UPDATE, {"settings": self._current_settings})) if chat_history and len(chat_history) > 0: for msg in chat_history.messages: - await self.input_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": msg})) + await self.send_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": msg})) + if create_response: + await self.send_buffer.put(SendEvents.RESPONSE_CREATE) @override async def close_session(self) -> None: """Close the session in the service.""" if self.peer_connection: - await self.peer_connection.close() - if self.data_channel: - await self.data_channel.close() + with contextlib.suppress(asyncio.CancelledError): + await self.peer_connection.close() self.peer_connection = None + if self.data_channel: + with contextlib.suppress(asyncio.CancelledError): + self.data_channel.close() self.data_channel = None - # region Event callbacks - - def response_audio_transcript_delta_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response audio transcript delta.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[StreamingTextContent(text=event.delta, choice_index=event.content_index)], - choice_index=event.content_index, - inner_content=event, - ), True - - def response_audio_transcript_done_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response audio transcript done.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[StreamingTextContent(text=event.transcript, choice_index=event.content_index)], - choice_index=event.content_index, - inner_content=event, - ), False - - def response_function_call_arguments_delta_callback( + async def _handle_function_call_arguments_done( self, event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response function call arguments delta.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[ - FunctionCallContent( - id=event.item_id, - name=event.call_id, - arguments=event.delta, - index=event.output_index, - metadata={"call_id": event.call_id}, - ) - ], - choice_index=0, - inner_content=event, - ), True - - def error_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> None: - """Handle error.""" - logger.error("Error received: %s", event.error) - - def session_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> None: - """Handle session.""" - logger.debug("Session created or updated, session: %s", event.session) - - async def response_function_call_arguments_done_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, ) -> None: """Handle response function call done.""" + plugin_name, function_name = self._call_id_to_function_map.pop(event.call_id, "-").split("-", 1) + if not plugin_name or not function_name: + logger.error("Function call needs to have a plugin name and function name") + return item = FunctionCallContent( id=event.item_id, - name=event.call_id, - arguments=event.delta, + plugin_name=plugin_name, + function_name=function_name, + arguments=event.arguments, index=event.output_index, metadata={"call_id": event.call_id}, ) - kernel: Kernel | None = kwargs.get("kernel") - call_id = item.name - function_name = next( - output_item_event.item.name - for output_item_event in self.event_log[ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED] - if output_item_event.item.call_id == call_id - ) - item.plugin_name, item.function_name = function_name.split("-", 1) - if kernel: - chat_history = ChatHistory() - await kernel.invoke_function_call(item, chat_history) - await self.input_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": chat_history.messages[-1]})) - # The model doesn't start responding to the tool call automatically, so triggering it here. - await self.input_buffer.put(SendEvents.RESPONSE_CREATE) - return chat_history.messages[-1], False + if not self.kernel and not self._current_settings.function_choice_behavior.auto_invoke_kernel_functions: + return + chat_history = ChatHistory() + await self.kernel.invoke_function_call(item, chat_history) + created_output = chat_history.messages[-1] + # This returns the output to the service + await self.send_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": created_output})) + # The model doesn't start responding to the tool call automatically, so triggering it here. + await self.send_buffer.put(SendEvents.RESPONSE_CREATE) + # This allows a user to have a full conversation in his code + await self.receive_buffer.put((ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, created_output)) + + async def get_ephemeral_token(self) -> str: + """Get an ephemeral token from OpenAI.""" + headers = {"Authorization": f"Bearer {self.client.api_key}", "Content-Type": "application/json"} + data = {"model": self.ai_model_id, "voice": "echo"} + + try: + async with ( + ClientSession() as session, + session.post( + f"{self.client.beta.realtime._client.base_url}/realtime/sessions", headers=headers, json=data + ) as response, + ): + if response.status not in [200, 201]: + error_text = await response.text() + raise Exception(f"Failed to get ephemeral token: {error_text}") - # region settings + result = await response.json() + return result["client_secret"]["value"] + + except Exception as e: + logger.error(f"Failed to get ephemeral token: {e!s}") + raise @override def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: @@ -823,3 +818,9 @@ def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"] ) return OpenAIRealtimeExecutionSettings + + @override + def _update_function_choice_settings_callback( + self, + ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: + return update_settings_from_function_call_configuration diff --git a/python/semantic_kernel/connectors/ai/open_ai/settings/open_ai_settings.py b/python/semantic_kernel/connectors/ai/open_ai/settings/open_ai_settings.py index 6423a5385a33..7276af4b1f3b 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/settings/open_ai_settings.py +++ b/python/semantic_kernel/connectors/ai/open_ai/settings/open_ai_settings.py @@ -32,6 +32,9 @@ class OpenAISettings(KernelBaseSettings): (Env var OPENAI_AUDIO_TO_TEXT_MODEL_ID) - text_to_audio_model_id: str | None - The OpenAI text to audio model ID to use, for example, jukebox-1. (Env var OPENAI_TEXT_TO_AUDIO_MODEL_ID) + - realtime_model_id: str | None - The OpenAI realtime model ID to use, + for example, gpt-4o-realtime-preview-2024-12-17. + (Env var OPENAI_REALTIME_MODEL_ID) - env_file_path: str | None - if provided, the .env settings are read from this file path location """ @@ -45,3 +48,4 @@ class OpenAISettings(KernelBaseSettings): text_to_image_model_id: str | None = None audio_to_text_model_id: str | None = None text_to_audio_model_id: str | None = None + realtime_model_id: str | None = None diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index c9a48f9d45b0..991854987faa 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -15,7 +15,6 @@ from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType -from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent from semantic_kernel.services.ai_service_client_base import AIServiceClientBase from semantic_kernel.utils.experimental_decorator import experimental_class @@ -29,8 +28,8 @@ class RealtimeClientBase(AIServiceClientBase, ABC): """Base class for a realtime client.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False - input_buffer: Queue[tuple[str, dict[str, Any]] | str] = Field(default_factory=Queue) - output_buffer: Queue[tuple[str, StreamingChatMessageContent]] = Field(default_factory=Queue) + send_buffer: Queue[str | tuple[str, Any]] = Field(default_factory=Queue) + receive_buffer: Queue[tuple[str, Any]] = Field(default_factory=Queue) async def __aenter__(self) -> "RealtimeClientBase": """Enter the context manager. diff --git a/python/semantic_kernel/connectors/ai/realtime_helpers.py b/python/semantic_kernel/connectors/ai/realtime_helpers.py new file mode 100644 index 000000000000..94549c402199 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/realtime_helpers.py @@ -0,0 +1,190 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging +from typing import Any, Final + +import numpy as np +import sounddevice as sd +from aiortc.mediastreams import MediaStreamError, MediaStreamTrack +from av.audio.frame import AudioFrame +from av.frame import Frame +from pydantic import Field, PrivateAttr + +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.kernel_pydantic import KernelBaseModel + +logger = logging.getLogger(__name__) + +SAMPLE_RATE: Final[int] = 48000 +TRACK_CHANNELS: Final[int] = 1 +PLAYER_CHANNELS: Final[int] = 2 +FRAME_DURATION: Final[int] = 20 +DTYPE: Final[np.dtype] = np.int16 + + +class SKAudioTrack(KernelBaseModel, MediaStreamTrack): + """A simple class using sounddevice to record audio from the default input device. + + And implementing the MediaStreamTrack interface for use with aiortc. + """ + + kind: str = "audio" + sample_rate: int = SAMPLE_RATE + channels: int = TRACK_CHANNELS + frame_duration: int = FRAME_DURATION + dtype: np.dtype = DTYPE + device: str | int | None = None + queue: asyncio.Queue[Frame] = Field(default_factory=asyncio.Queue) + is_recording: bool = False + stream: sd.InputStream | None = None + frame_size: int = 0 + _recording_task: asyncio.Task | None = None + _loop: asyncio.AbstractEventLoop | None = None + _pts: int = 0 # Add this to track the pts + + def __init__(self, **kwargs: Any): + """Initialize the audio track. + + Args: + **kwargs: Additional keyword arguments. + + """ + kwargs["frame_size"] = int( + kwargs.get("sample_rate", SAMPLE_RATE) * kwargs.get("frame_duration", FRAME_DURATION) / 1000 + ) + super().__init__(**kwargs) + MediaStreamTrack.__init__(self) + + async def recv(self) -> Frame: + """Receive the next frame of audio data.""" + if not self._recording_task: + self._recording_task = asyncio.create_task(self.start_recording()) + + try: + return await self.queue.get() + except Exception as e: + logger.error(f"Error receiving audio frame: {e!s}") + raise MediaStreamError("Failed to receive audio frame") + + async def start_recording(self): + """Start recording audio from the input device.""" + if self.is_recording: + return + + self.is_recording = True + self._loop = asyncio.get_running_loop() + self._pts = 0 # Reset pts when starting recording + + try: + + def callback(indata: np.ndarray, frames: int, time: Any, status: Any) -> None: + if status: + logger.warning(f"Audio input status: {status}") + + audio_data = indata.copy() + if audio_data.dtype != self.dtype: + if self.dtype == np.int16: + audio_data = (audio_data * 32767).astype(self.dtype) + else: + audio_data = audio_data.astype(self.dtype) + + frame = AudioFrame( + format="s16", + layout="mono", + samples=len(audio_data), + ) + frame.rate = self.sample_rate + frame.pts = self._pts + frame.planes[0].update(audio_data.tobytes()) + self._pts += len(audio_data) + if self._loop and self._loop.is_running(): + asyncio.run_coroutine_threadsafe(self.queue.put(frame), self._loop) + + self.stream = sd.InputStream( + device=self.device, + channels=self.channels, + samplerate=self.sample_rate, + dtype=self.dtype, + blocksize=self.frame_size, + callback=callback, + ) + self.stream.start() + + while self.is_recording: + await asyncio.sleep(0.1) + + except Exception as e: + logger.error(f"Error in audio recording: {e!s}") + raise + finally: + self.is_recording = False + + +class SKSimplePlayer(KernelBaseModel): + """Simple class that plays audio using sounddevice. + + Make sure the device_id is set to the correct device for your system. + + The sample rate, channels and frame duration should be set to match the audio you + are receiving, the defaults are for WebRTC. + """ + + device_id: int | None = None + sample_rate: int = SAMPLE_RATE + channels: int = PLAYER_CHANNELS + frame_duration_ms: int = FRAME_DURATION + queue: asyncio.Queue[np.ndarray] = Field(default_factory=asyncio.Queue) + _stream: sd.OutputStream | None = PrivateAttr(None) + + def model_post_init(self, __context: Any) -> None: + """Initialize the audio stream.""" + self._stream = sd.OutputStream( + callback=self.callback, + samplerate=self.sample_rate, + channels=self.channels, + dtype=np.int16, + blocksize=int(self.sample_rate * self.frame_duration_ms / 1000), + device=self.device_id, + ) + + async def __aenter__(self): + """Start the audio stream when entering a context.""" + self.start() + return self + + async def __aexit__(self, exc_type, exc, tb): + """Stop the audio stream when exiting a context.""" + self.stop() + + def start(self): + """Start the audio stream.""" + if self._stream: + self._stream.start() + + def stop(self): + """Stop the audio stream.""" + if self._stream: + self._stream.stop() + + def callback(self, outdata, frames, time, status): + """This callback is called by sounddevice when it needs more audio data to play.""" + if status: + logger.info(f"Audio output status: {status}") + if self.queue.empty(): + return + data: np.ndarray = self.queue.get_nowait() + outdata[:] = data.reshape(outdata.shape) + + async def realtime_client_callback(self, frame: AudioFrame): + """This function is used by the RealtimeClientBase to play audio.""" + await self.queue.put(frame.to_ndarray()) + + async def add_audio(self, audio_content: AudioContent): + """This function is used to add audio to the queue for playing. + + It uses a shortcut for this sample, because we know a AudioFrame is in the inner_content field. + """ + if audio_content.inner_content and isinstance(audio_content.inner_content, AudioFrame): + await self.queue.put(audio_content.inner_content.to_ndarray()) + # TODO (eavanvalkenburg): check ndarray diff --git a/python/semantic_kernel/contents/audio_content.py b/python/semantic_kernel/contents/audio_content.py index 77d4a9970a63..f52b467780b4 100644 --- a/python/semantic_kernel/contents/audio_content.py +++ b/python/semantic_kernel/contents/audio_content.py @@ -43,7 +43,7 @@ class AudioContent(BinaryContent): tag: ClassVar[str] = AUDIO_CONTENT_TAG @classmethod - def from_audio_file(cls: type[_T], path: str) -> "AudioContent": + def from_audio_file(cls: type[_T], path: str) -> _T: """Create an instance from an audio file.""" mime_type = mimetypes.guess_type(path)[0] with open(path, "rb") as audio_file: @@ -54,6 +54,6 @@ def to_dict(self) -> dict[str, Any]: return {"type": "audio_url", "audio_url": {"uri": str(self)}} @classmethod - def from_nd_array(cls: type[_T], data: ndarray, mime_type: str) -> "AudioContent": + def from_nd_array(cls: type[_T], data: ndarray, mime_type: str) -> _T: """Create an instance from an nd array.""" return cls(data=data, mime_type=mime_type) diff --git a/python/semantic_kernel/contents/binary_content.py b/python/semantic_kernel/contents/binary_content.py index 7ac5840dd8fb..53a4fa5e2bde 100644 --- a/python/semantic_kernel/contents/binary_content.py +++ b/python/semantic_kernel/contents/binary_content.py @@ -69,25 +69,25 @@ def __init__( ai_model_id (str | None): The id of the AI model that generated this response. metadata (dict[str, Any]): Any metadata that should be attached to the response. """ - _data_uri = None + data_uri_ = None if data_uri: - _data_uri = DataUri.from_data_uri(data_uri, self.default_mime_type) + data_uri_ = DataUri.from_data_uri(data_uri, self.default_mime_type) if "metadata" in kwargs: - kwargs["metadata"].update(_data_uri.parameters) + kwargs["metadata"].update(data_uri_.parameters) else: - kwargs["metadata"] = _data_uri.parameters - elif data: + kwargs["metadata"] = data_uri_.parameters + elif data is not None: match data: case str(): - _data_uri = DataUri( + data_uri_ = DataUri( data_str=data, data_format=data_format, mime_type=mime_type or self.default_mime_type ) case bytes(): - _data_uri = DataUri( + data_uri_ = DataUri( data_bytes=data, data_format=data_format, mime_type=mime_type or self.default_mime_type ) case ndarray(): - _data_uri = DataUri(data_array=data, mime_type=mime_type or self.default_mime_type) + data_uri_ = DataUri(data_array=data, mime_type=mime_type or self.default_mime_type) if uri is not None: if isinstance(uri, str) and os.path.exists(uri): @@ -96,7 +96,7 @@ def __init__( uri = Url(uri) super().__init__(uri=uri, **kwargs) - self._data_uri = _data_uri + self._data_uri = data_uri_ @computed_field # type: ignore @property @@ -115,7 +115,7 @@ def data_uri(self, value: str): @property def data(self) -> bytes | ndarray: """Get the data.""" - if self._data_uri and self._data_uri.data_array: + if self._data_uri and self._data_uri.data_array is not None: return self._data_uri.data_array if self._data_uri and self._data_uri.data_bytes: return self._data_uri.data_bytes diff --git a/python/semantic_kernel/contents/utils/data_uri.py b/python/semantic_kernel/contents/utils/data_uri.py index d70c175209fa..371d64206972 100644 --- a/python/semantic_kernel/contents/utils/data_uri.py +++ b/python/semantic_kernel/contents/utils/data_uri.py @@ -48,8 +48,13 @@ def update_data(self, value: str | bytes | ndarray): @classmethod def _validate_data(cls, values: Any) -> dict[str, Any]: """Validate the data.""" - if isinstance(values, dict) and not values.get("data_bytes") and not values.get("data_str"): - raise ContentInitializationError("Either data_bytes or data_str must be provided.") + if ( + isinstance(values, dict) + and not values.get("data_bytes") + and not values.get("data_str") + and values.get("data_array") is None + ): + raise ContentInitializationError("Either data_bytes, data_str or data_array must be provided.") return values @model_validator(mode="after") @@ -59,7 +64,7 @@ def _parse_data(self) -> Self: Will try to decode the data bytes to a string if it is not already set. However if the data array is used, it will not be converted to a string. """ - if self.data_array: + if self.data_array is not None: return self if not self.data_str and self.data_bytes: if self.data_format and self.data_format.lower() == "base64": diff --git a/python/uv.lock b/python/uv.lock index fde73e9c4f0a..bb5da1b98ae9 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -445,7 +445,7 @@ name = "build" version = "1.2.2.post1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(os_name == 'nt' and sys_platform == 'darwin') or (os_name == 'nt' and sys_platform == 'linux') or (os_name == 'nt' and sys_platform == 'win32')" }, + { name = "colorama", marker = "os_name == 'nt' and sys_platform == 'win32'" }, { name = "importlib-metadata", marker = "(python_full_version < '3.10.2' and sys_platform == 'darwin') or (python_full_version < '3.10.2' and sys_platform == 'linux') or (python_full_version < '3.10.2' and sys_platform == 'win32')" }, { name = "packaging", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "pyproject-hooks", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -688,7 +688,7 @@ name = "click" version = "8.1.8" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(platform_system == 'Windows' and sys_platform == 'darwin') or (platform_system == 'Windows' and sys_platform == 'linux') or (platform_system == 'Windows' and sys_platform == 'win32')" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/b9/2e/0090cbf739cee7d23781ad4b89a9894a41538e4fcf4c31dcdd705b78eb8b/click-8.1.8.tar.gz", hash = "sha256:ed53c9d8990d83c2a27deae68e4ee337473f6330c040a31d4225c9574d16096a", size = 226593 } wheels = [ @@ -1837,7 +1837,7 @@ name = "ipykernel" version = "6.29.5" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "appnope", marker = "(platform_system == 'Darwin' and sys_platform == 'darwin') or (platform_system == 'Darwin' and sys_platform == 'linux') or (platform_system == 'Darwin' and sys_platform == 'win32')" }, + { name = "appnope", marker = "sys_platform == 'darwin'" }, { name = "comm", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "debugpy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "ipython", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -2744,7 +2744,7 @@ name = "nvidia-cudnn-cu12" version = "9.1.0.70" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/9f/fd/713452cd72343f682b1c7b9321e23829f00b842ceaedcda96e742ea0b0b3/nvidia_cudnn_cu12-9.1.0.70-py3-none-manylinux2014_x86_64.whl", hash = "sha256:165764f44ef8c61fcdfdfdbe769d687e06374059fbb388b6c89ecb0e28793a6f", size = 664752741 }, @@ -2755,7 +2755,7 @@ name = "nvidia-cufft-cu12" version = "11.2.1.3" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/27/94/3266821f65b92b3138631e9c8e7fe1fb513804ac934485a8d05776e1dd43/nvidia_cufft_cu12-11.2.1.3-py3-none-manylinux2014_x86_64.whl", hash = "sha256:f083fc24912aa410be21fa16d157fed2055dab1cc4b6934a0e03cba69eb242b9", size = 211459117 }, @@ -2774,9 +2774,9 @@ name = "nvidia-cusolver-cu12" version = "11.6.1.9" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-cublas-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-cublas-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/3a/e1/5b9089a4b2a4790dfdea8b3a006052cfecff58139d5a4e34cb1a51df8d6f/nvidia_cusolver_cu12-11.6.1.9-py3-none-manylinux2014_x86_64.whl", hash = "sha256:19e33fa442bcfd085b3086c4ebf7e8debc07cfe01e11513cc6d332fd918ac260", size = 127936057 }, @@ -2787,7 +2787,7 @@ name = "nvidia-cusparse-cu12" version = "12.3.1.170" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "nvidia-nvjitlink-cu12", marker = "sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/db/f7/97a9ea26ed4bbbfc2d470994b8b4f338ef663be97b8f677519ac195e113d/nvidia_cusparse_cu12-12.3.1.170-py3-none-manylinux2014_x86_64.whl", hash = "sha256:ea4f11a2904e2a8dc4b1833cc1b5181cde564edd0d5cd33e3c168eff2d1863f1", size = 207454763 }, @@ -3408,7 +3408,7 @@ name = "portalocker" version = "2.10.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "pywin32", marker = "(platform_system == 'Windows' and sys_platform == 'darwin') or (platform_system == 'Windows' and sys_platform == 'linux') or (platform_system == 'Windows' and sys_platform == 'win32')" }, + { name = "pywin32", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/ed/d3/c6c64067759e87af98cc668c1cc75171347d0f1577fab7ca3749134e3cd4/portalocker-2.10.1.tar.gz", hash = "sha256:ef1bf844e878ab08aee7e40184156e1151f228f103aa5c6bd0724cc330960f8f", size = 40891 } wheels = [ @@ -4784,7 +4784,7 @@ hugging-face = [ { name = "transformers", extra = ["torch"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] milvus = [ - { name = "milvus", marker = "(platform_system != 'Windows' and sys_platform == 'darwin') or (platform_system != 'Windows' and sys_platform == 'linux') or (platform_system != 'Windows' and sys_platform == 'win32')" }, + { name = "milvus", marker = "sys_platform == 'darwin' or sys_platform == 'linux'" }, { name = "pymilvus", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] mistralai = [ @@ -4869,7 +4869,7 @@ requires-dist = [ { name = "google-generativeai", marker = "extra == 'google'", specifier = "~=0.7" }, { name = "ipykernel", marker = "extra == 'notebooks'", specifier = "~=6.29" }, { name = "jinja2", specifier = "~=3.1" }, - { name = "milvus", marker = "platform_system != 'Windows' and extra == 'milvus'", specifier = ">=2.3,<2.3.8" }, + { name = "milvus", marker = "sys_platform != 'win32' and extra == 'milvus'", specifier = ">=2.3,<2.3.8" }, { name = "mistralai", marker = "extra == 'mistralai'", specifier = ">=1.2,<2.0" }, { name = "motor", marker = "extra == 'mongo'", specifier = ">=3.3.2,<3.7.0" }, { name = "nest-asyncio", specifier = "~=1.6" }, @@ -5272,21 +5272,21 @@ dependencies = [ { name = "fsspec", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "jinja2", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "networkx", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "nvidia-cublas-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cuda-cupti-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cuda-nvrtc-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cuda-runtime-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cudnn-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cufft-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-curand-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cusolver-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-cusparse-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-nccl-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-nvjitlink-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, - { name = "nvidia-nvtx-cu12", marker = "(platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "nvidia-cublas-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-cupti-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-nvrtc-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cuda-runtime-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cudnn-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cufft-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-curand-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusolver-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-cusparse-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nccl-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvjitlink-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, + { name = "nvidia-nvtx-cu12", marker = "platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "setuptools", marker = "(python_full_version >= '3.12' and sys_platform == 'darwin') or (python_full_version >= '3.12' and sys_platform == 'linux') or (python_full_version >= '3.12' and sys_platform == 'win32')" }, { name = "sympy", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, - { name = "triton", marker = "(python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'darwin') or (python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'linux') or (python_full_version < '3.13' and platform_machine == 'x86_64' and platform_system == 'Linux' and sys_platform == 'win32')" }, + { name = "triton", marker = "python_full_version < '3.13' and platform_machine == 'x86_64' and sys_platform == 'linux'" }, { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] wheels = [ @@ -5328,7 +5328,7 @@ name = "tqdm" version = "4.67.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "colorama", marker = "(platform_system == 'Windows' and sys_platform == 'darwin') or (platform_system == 'Windows' and sys_platform == 'linux') or (platform_system == 'Windows' and sys_platform == 'win32')" }, + { name = "colorama", marker = "sys_platform == 'win32'" }, ] sdist = { url = "https://files.pythonhosted.org/packages/a8/4b/29b4ef32e036bb34e4ab51796dd745cdba7ed47ad142a9f4a1eb8e0c744d/tqdm-4.67.1.tar.gz", hash = "sha256:f8aef9c52c08c13a65f30ea34f4e5aac3fd1a34959879d7e59e63027286627f2", size = 169737 } wheels = [ @@ -5376,7 +5376,7 @@ name = "triton" version = "3.1.0" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "filelock", marker = "(python_full_version < '3.13' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform == 'win32')" }, + { name = "filelock", marker = "python_full_version < '3.13' and sys_platform == 'linux'" }, ] wheels = [ { url = "https://files.pythonhosted.org/packages/98/29/69aa56dc0b2eb2602b553881e34243475ea2afd9699be042316842788ff5/triton-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b0dd10a925263abbe9fa37dcde67a5e9b2383fc269fdf59f5657cac38c5d1d8", size = 209460013 }, From 5924ba6088fe0befa8c93137b283a1142f9a8c0d Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 15:55:51 +0100 Subject: [PATCH 10/25] added dependency --- python/pyproject.toml | 5 +- .../audio/04-chat_with_realtime_api.py | 5 + .../open_ai/services/open_ai_realtime_base.py | 4 +- python/uv.lock | 135 ++++++++++++++++++ 4 files changed, 145 insertions(+), 4 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 67b762bfd78d..a88f332c8cd1 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -126,7 +126,8 @@ dapr = [ "flask-dapr>=1.14.0" ] openai_realtime = [ - "openai[realtime] ~= 1.0" + "openai[realtime] ~= 1.0", + "aiortc>=1.9.0" ] [tool.uv] @@ -225,5 +226,3 @@ name = "semantic_kernel" [build-system] requires = ["flit-core >= 3.9,<4.0"] build-backend = "flit_core.buildapi" - - diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index 0f895f7dc9dc..902ad72d48d4 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -156,6 +156,11 @@ async def main() -> None: await realtime_client.update_session( settings=settings, chat_history=chat_history, kernel=kernel, create_response=True ) + # you can also send other events to the service, like this + # await realtime_client.send_buffer.put(( + # SendEvents.CONVERSATION_ITEM_CREATE, + # {"item": ChatMessageContent(role="user", content="Hi there, who are you?")}, + # )) async with asyncio.TaskGroup() as tg: tg.create_task(realtime_client.start_streaming()) tg.create_task(stream_handler.listen()) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py index e387ef4005aa..7caf5a5671df 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -529,7 +529,7 @@ async def _on_track(self, track: MediaStreamTrack) -> None: async def _on_data(self, data: str) -> None: """This method is called whenever a data channel message is received. - The data is parsed into a RealtimeServerEvent (by OpenAI) and then processed. + The data is parsed into a RealtimeServerEvent (by OpenAI code) and then processed. """ try: event = cast( @@ -580,6 +580,8 @@ async def _on_data(self, data: str) -> None: case _: logger.debug(f"Received event: {event}") # we put all event in the output buffer, but after the interpreted one. + # so when dealing with them, make sure to check the type of the event, since they + # might be of different types. await self.receive_buffer.put((event.type, event)) @override diff --git a/python/uv.lock b/python/uv.lock index bb5da1b98ae9..d1d02c8ce8b1 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -125,6 +125,50 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b8/62/c9fa5bafe03186a0e4699150a7fed9b1e73240996d0d2f0e5f70f3fdf471/aiohttp-3.11.11-cp313-cp313-win_amd64.whl", hash = "sha256:c7a06301c2fb096bdb0bd25fe2011531c1453b9f2c163c8031600ec73af1cc99", size = 436081 }, ] +[[package]] +name = "aioice" +version = "0.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "dnspython", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "ifaddr", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/33/b6/e2b0e48ccb5b04fe29265e93f14a0915f416e359c897ae87d570566c430b/aioice-0.9.0.tar.gz", hash = "sha256:fc2401b1c4b6e19372eaaeaa28fd1bd9cbf6b0e412e48625297c53b495eebd1e", size = 40324 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/b6/35/d21e48d3ba25d32aba5d142d54c4491376c659dd74d052a30dd25198007b/aioice-0.9.0-py3-none-any.whl", hash = "sha256:b609597a3a5a611e0004ff04772e16aceb881d51c25c0afc4ceac05d5e50024e", size = 24177 }, +] + +[[package]] +name = "aiortc" +version = "1.9.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "aioice", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "av", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "cffi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "google-crc32c", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pyee", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pylibsrtp", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "pyopenssl", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/71/32/e9b01e2271124643e5dc15c273f2bb8155efebf5bc2115407441ac62f4c5/aiortc-1.9.0.tar.gz", hash = "sha256:03faa76d76ef0e5989ac10386898b029369756102217230e2fcd4b029c50b303", size = 1168973 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/93/01/db89910fc4dfb72ca25fd9a41326762a490d93d39d2fc4aac3f86c05857d/aiortc-1.9.0-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:e3e67c1970c2cffacac53c8f161df264efc62b22721c64a621940935028ee087", size = 1216069 }, + { url = "https://files.pythonhosted.org/packages/4c/6d/76ed96521080492c7264eacf73a8cba2202f1ff9f59af1776c5a2532f332/aiortc-1.9.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:d893cb3d4ffa0ff4f9bb03a88f0a700cdbcd4c0dc060a46c59a27ccd1c890663", size = 896012 }, + { url = "https://files.pythonhosted.org/packages/8c/87/1f666108764fa5b557bed4f0fd5e2acccd739bb2cca2b766dcacb53e5669/aiortc-1.9.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:176b4eb38d833667f87cf719a7a3e105e25a35b138b30893294418c1c96e38db", size = 1779113 }, + { url = "https://files.pythonhosted.org/packages/32/03/f3233e936f7a81549bd95f33f3d304e2a9211cb35d819d74570c0718b1ac/aiortc-1.9.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c44b610f36b8d17123855dfbe915fa6874201765b8a2c7fd9cf72d14cf417740", size = 1896322 }, + { url = "https://files.pythonhosted.org/packages/96/99/6672cf57777801c6ddacc13e1ee07f8c2151d0847a4f81455eeec998eaed/aiortc-1.9.0-cp38-abi3-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:55505adb31d56cba19a1ef8ad6aa9b727ccdba2a83bfbfb4aa79ef3c472026a6", size = 1918600 }, + { url = "https://files.pythonhosted.org/packages/76/e3/bdb76e7e51bc4fc7a5869597de2effad073ccf5ef14de3aed742d7384107/aiortc-1.9.0-cp38-abi3-win32.whl", hash = "sha256:680b703e35870e301535c930bfe32e7d012224a91ce51531aba45a3124ef07cc", size = 923055 }, + { url = "https://files.pythonhosted.org/packages/6a/df/de098b31a3fbf1117f6d4cb84c14518636054e3c95a9d9f693a1123c95b3/aiortc-1.9.0-cp38-abi3-win_amd64.whl", hash = "sha256:de5e7020cfc2d2d9fb95690926ff2e3b3c30cd4f5f5bc68d5b6756a8eebb686e", size = 1009610 }, + { url = "https://files.pythonhosted.org/packages/95/26/c382db590897fe638254f948d8514772d13ff59b5ada0a71d87322f48c52/aiortc-1.9.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:34c516ae4e70e8f64494305057af09311444325722fe6938ec38dd1e111adca9", size = 1209093 }, + { url = "https://files.pythonhosted.org/packages/68/48/2fe7de04461fdc4aee8c78c67cfe03579eaa72fb215c4b063acaeb4fd118/aiortc-1.9.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:40e61c1b84914d6f4c2968ff49353a22eed9419de74b151237cdb71af431209c", size = 888818 }, + { url = "https://files.pythonhosted.org/packages/da/d5/94bf7ed6189c316ffef930787cba009387f9bcd2f1c482392b71cca3918d/aiortc-1.9.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1924e130a441507b1315956aff05c504a274f1a09802def225d0f3a3d1870320", size = 1732549 }, + { url = "https://files.pythonhosted.org/packages/e7/0a/6495c696cd7f806bafe511fb27203ce918947c4461398384a4e6bd4b7e57/aiortc-1.9.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cbb62950e396c311e398925149fa76bc90b8d6525b4eccf28cba704e7ded8bf5", size = 1843911 }, + { url = "https://files.pythonhosted.org/packages/82/36/ffd0f74c73fa6abca0b76bd38473ed7d82dfbada7e57c6efe2a37ee40483/aiortc-1.9.0-pp310-pypy310_pp73-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:5234177e8d3126a0190ed9b6f8d0288daedcc0158c45cc279b4e6ac7d97f43f8", size = 1868240 }, + { url = "https://files.pythonhosted.org/packages/fb/46/8cb087a11f2f2d1139bd7e21615cc082097bffc4990d43c9f45f9cf6c8bf/aiortc-1.9.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:0e31575eb050aa68e0ea4c519aef101770b2297954f49e64a5c3d73ef27702ea", size = 1004186 }, +] + [[package]] name = "aiosignal" version = "1.3.2" @@ -239,6 +283,38 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/87/1f/bc95e43ffb57c05b8efcc376dd55a0240bf58f47ddf5a0f92452b6457b75/Authlib-1.3.1-py2.py3-none-any.whl", hash = "sha256:d35800b973099bbadc49b42b256ecb80041ad56b7fe1216a362c7943c088f377", size = 223827 }, ] +[[package]] +name = "av" +version = "12.3.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/00/f8/5adeeae0c42a7130933d168b8d84a21c98a32cb9fcf9222e2541ed0d9c7b/av-12.3.0.tar.gz", hash = "sha256:04b1892562aff3277efc79f32bd8f1d0cbb64ed011241cb3e96f9ad471816c22", size = 3833953 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/53/57/414fe243152ef3f5a364f3e0137c16fbfe67c3f096eac1dc49d614de8f98/av-12.3.0-cp310-cp310-macosx_10_13_x86_64.whl", hash = "sha256:b3b1fe6b5ab9af2d09dcdcc5473a3523f7162c3fa0c6b3c379b697fede1e88a5", size = 24663048 }, + { url = "https://files.pythonhosted.org/packages/15/e8/8795c6cf7d4ef34b30690b3e1601982c6ce9ec8c42a681fff5791a4c4ca9/av-12.3.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:b5f92ba67dca9bac8ce955b09d41e7e92977199adbd0f2aff02653bb40b0ac16", size = 19930356 }, + { url = "https://files.pythonhosted.org/packages/f9/90/6e0340af495b1028be90fae4793900df9853732e38003a795a14bb52dee5/av-12.3.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3389eebd1f5bb36ebfaa8441c65c14d7433b354d91f9dbb08a6e6225d16a7226", size = 31623727 }, + { url = "https://files.pythonhosted.org/packages/0a/d1/34d69a00405e0c58059431b24e8abbf2861446b740eb1813c1569a0b7467/av-12.3.0-cp310-cp310-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:385b27638bc56fd1560be3b9e86b5cc843cae931503a02e6e504c0357176873e", size = 31126299 }, + { url = "https://files.pythonhosted.org/packages/0a/5f/5ab859d8770ac1203d492e418cf949cfcac5c25994e9754c536fb37578fc/av-12.3.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0220fce2a62d71cc5e89617419b6224ddb43f1753b00f68b5c9af8b5f41d38c9", size = 33490936 }, + { url = "https://files.pythonhosted.org/packages/39/6f/46a468053c8ae594c91a385f2323ade83746e03ba11ba14fb79db61a23ff/av-12.3.0-cp310-cp310-win_amd64.whl", hash = "sha256:8328c90f783b3392279a2d3a79789267691f5e5f7c4a160990a41194d268ec59", size = 25973279 }, + { url = "https://files.pythonhosted.org/packages/5d/20/256fa4fc4ef9bb46fdc4be4662e13a30b0334487c955961f3816d94db04b/av-12.3.0-cp311-cp311-macosx_10_13_x86_64.whl", hash = "sha256:cc06a806419fddc7102150ffe353c7d96b99b95fd12864280c91c851603fd4cb", size = 24658122 }, + { url = "https://files.pythonhosted.org/packages/5d/45/a9d0475539b4f49deb34f3da558de31cefc6be867d5c0603d575a8485069/av-12.3.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:8e2130ff622a574d3d5d6e88ac335efcdd98c375bb341f87d9fe540830a746f5", size = 19923068 }, + { url = "https://files.pythonhosted.org/packages/af/27/1f2b3e46059c6618fd76ba12a96b49dc8515a426cd477032cd33f80505e8/av-12.3.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8e8b9bd99f916ff4d1278654e94658e6ace7ca60f6321f254d09c8cd81d9095b", size = 32555100 }, + { url = "https://files.pythonhosted.org/packages/28/34/759741d397a8bdbb8a359b8b5d49832a444b26c9a7f79c0f88be76a6b979/av-12.3.0-cp311-cp311-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:9e375d1d89a5c6edfd9f66701fdb6cc9161cc1ff99d15ff0bda21ee1ad38e9e0", size = 31936355 }, + { url = "https://files.pythonhosted.org/packages/b4/6e/77426cb92117c941b0f759908bc83f34f259b11b353acb5de95972b452f7/av-12.3.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ef9066fd8d86548e12d587cbfe7b852159e48ff3c732271c3032668d4bd7c599", size = 34416598 }, + { url = "https://files.pythonhosted.org/packages/ff/d3/4b0fddcd54d0a88ee7e035f239ebb56ce139fac8e02ee0942c43746a66ff/av-12.3.0-cp311-cp311-win_amd64.whl", hash = "sha256:bfaa9864560e43d45d254ed95f70ab1aab24a2fa0cc35ac99eef362f1453bec0", size = 25975217 }, + { url = "https://files.pythonhosted.org/packages/e4/c1/0636bccf5a1a2c935952614b9d34d8d8aae078c9773a60efb5376702f499/av-12.3.0-cp312-cp312-macosx_10_13_x86_64.whl", hash = "sha256:5174e995772ebe33561980dca625f830aea8d39a4338728dedb41ae7dc2605af", size = 24669628 }, + { url = "https://files.pythonhosted.org/packages/ef/7d/9126abdafe20fa73d2c19fd108450363253cfea283c350618cc1434f473c/av-12.3.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:028d8b40308536f740dace3efd0178eb96825b414897c9594fb74136532901cb", size = 19928928 }, + { url = "https://files.pythonhosted.org/packages/27/75/c1b9e0aa4bd0d8b8311f366b6b38f6c6600d66baddfe2888accc7f76b1f5/av-12.3.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b030791ecc6185776d832d19ce196f61daf3e17e591a9bb6fd181280e1754138", size = 32793461 }, + { url = "https://files.pythonhosted.org/packages/5a/06/1364c445f8a8ab4870f0f5c4530b496257ae09de7fa01b6108525abea8b9/av-12.3.0-cp312-cp312-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a3703a35481fda5798a27bf6208c1ec3b61c18931625771fb3c9fd870539c7d7", size = 32217647 }, + { url = "https://files.pythonhosted.org/packages/27/08/220d5a1ae7e7830d66d041c71e607c1f5df2e3598b12fb406b0d7c2defa7/av-12.3.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:32f3eef56b2df289db6105f9fe2ebc9a8134a8adbd62190daeb8e22c4ff47794", size = 34746451 }, + { url = "https://files.pythonhosted.org/packages/96/67/9f1c444864d4f3e3773100b9ed20e670f80d5575b7a8fd53cca20de9d681/av-12.3.0-cp312-cp312-win_amd64.whl", hash = "sha256:62d036ee8321d67190887012c3dbcd1ad83248603cc29ea75fbb75835b8d6e6e", size = 25977611 }, + { url = "https://files.pythonhosted.org/packages/e2/63/e1b22a63404a22bf49a981e2386f33a2d7fd7c1fe1087cca34cc06652b40/av-12.3.0-pp310-pypy310_pp73-macosx_10_15_x86_64.whl", hash = "sha256:e47ba817fcd46c9f2c94d638abcdeda120adedcd09605984a5cee844f739a833", size = 24271362 }, + { url = "https://files.pythonhosted.org/packages/64/08/16c8a6a0a1df2a651c0124368e470df85f3086cf98624f6698706f91e717/av-12.3.0-pp310-pypy310_pp73-macosx_11_0_arm64.whl", hash = "sha256:b456cbb7ddd252f0f2db06a09dc10ade201e82e0eb8d3a7b609689907b2802df", size = 19575368 }, + { url = "https://files.pythonhosted.org/packages/eb/6b/18369c3cb78f6aaadcbf7c94683d75c2cefaf79962016ffbf6d0d1b21b22/av-12.3.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:50ccb92605d59732d2a2923786a5dba746a98c5fd6b4d30a5975785673c42c9e", size = 23344574 }, + { url = "https://files.pythonhosted.org/packages/40/61/f26be7deb3675f15925f6006d9f0a2937a5cb15a176b32935eaac8ecaeff/av-12.3.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:061b15203f22e95c60b1cc14702618acbf18e976cf3144298e2f6dc89b7aa993", size = 23272262 }, + { url = "https://files.pythonhosted.org/packages/e9/3f/fb6ac8f1df45ff06155e0850e53d944536966d0564e0b0f5b839e67352cb/av-12.3.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:65849ca4e54f2d50ed263ab488ef051bd973cbdbe2a7c947b31ff965bb7bfddd", size = 25186971 }, + { url = "https://files.pythonhosted.org/packages/94/d7/7b1a9b9c2321cb0dcd093d6dca6a038c5bef27784fb5a58d2798a56459cf/av-12.3.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:18e915ca9001f9491cb4091fe6ca0744a48da20412be44f71bbfc641efbf518f", size = 25757707 }, +] + [[package]] name = "azure-ai-inference" version = "1.0.0b7" @@ -1802,6 +1878,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/76/c6/c88e154df9c4e1a2a66ccf0005a88dfb2650c1dffb6f5ce603dfbd452ce3/idna-3.10-py3-none-any.whl", hash = "sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3", size = 70442 }, ] +[[package]] +name = "ifaddr" +version = "0.2.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/e8/ac/fb4c578f4a3256561548cd825646680edcadb9440f3f68add95ade1eb791/ifaddr-0.2.0.tar.gz", hash = "sha256:cc0cbfcaabf765d44595825fb96a99bb12c79716b73b44330ea38ee2b0c4aed4", size = 10485 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/9c/1f/19ebc343cc71a7ffa78f17018535adc5cbdd87afb31d7c34874680148b32/ifaddr-0.2.0-py3-none-any.whl", hash = "sha256:085e0305cfe6f16ab12d72e2024030f5d52674afad6911bb1eee207177b8a748", size = 12314 }, +] + [[package]] name = "importlib-metadata" version = "8.5.0" @@ -3874,6 +3959,18 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b4/46/93416fdae86d40879714f72956ac14df9c7b76f7d41a4d68aa9f71a0028b/pydantic_settings-2.7.1-py3-none-any.whl", hash = "sha256:590be9e6e24d06db33a4262829edef682500ef008565a969c73d39d5f8bfb3fd", size = 29718 }, ] +[[package]] +name = "pyee" +version = "12.1.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "typing-extensions", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/0a/37/8fb6e653597b2b67ef552ed49b438d5398ba3b85a9453f8ada0fd77d455c/pyee-12.1.1.tar.gz", hash = "sha256:bbc33c09e2ff827f74191e3e5bbc6be7da02f627b7ec30d86f5ce1a6fb2424a3", size = 30915 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/25/68/7e150cba9eeffdeb3c5cecdb6896d70c8edd46ce41c0491e12fb2b2256ff/pyee-12.1.1-py3-none-any.whl", hash = "sha256:18a19c650556bb6b32b406d7f017c8f513aceed1ef7ca618fb65de7bd2d347ef", size = 15527 }, +] + [[package]] name = "pygments" version = "2.19.1" @@ -3897,6 +3994,29 @@ crypto = [ { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] +[[package]] +name = "pylibsrtp" +version = "0.10.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/6b/ae/c95199144eed954976223bdce3f94564eb6c43567111aff8048a26a429bd/pylibsrtp-0.10.0.tar.gz", hash = "sha256:d8001912d7f51bd05b4ea3551747930631777fd37892cf3bfe0e541a742e699f", size = 10557 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/1c/d2/ffc24f80e83a54d9b309cdae6b31cf9294b4f3a85ab107827fd272d1e687/pylibsrtp-0.10.0-cp38-abi3-macosx_10_9_x86_64.whl", hash = "sha256:6a1121ceea3339e0a84842a4a9da0fcf57cc8f99eb60dbf31a46d978b4170e7c", size = 1704188 }, + { url = "https://files.pythonhosted.org/packages/66/3e/db86a09a5cb290a274f76ce25f4fae3a7e3c4a4dbc64baf7e2aaa57a32bb/pylibsrtp-0.10.0-cp38-abi3-macosx_11_0_arm64.whl", hash = "sha256:ca1994e73c6857b0a695fdde94cc5ac846c1b0d5d8766255a1dc2db40857f667", size = 2028580 }, + { url = "https://files.pythonhosted.org/packages/21/ab/9b2b5ad2ceaa1660de16e0a2e3c54a2043a9c4a3eef7718930c78dc84e77/pylibsrtp-0.10.0-cp38-abi3-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bb7640b524544603d07bd4373b04c9582c8cfe41d9789d3f492081f053bed9c1", size = 2484470 }, + { url = "https://files.pythonhosted.org/packages/ab/e6/b0a30e79aa2312834b33f5e9c0ad459fc94e195c610634ee9665fafb1fc8/pylibsrtp-0.10.0-cp38-abi3-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f13aa945e1dcf8c138bf3d4a6e34056c4c2f69bf9934bc53b320ef14c7317ccc", size = 2078367 }, + { url = "https://files.pythonhosted.org/packages/16/78/9ea0c88490ad4fe9683ddf3bbee702c7a2331e83a333bb3aa52e8d7d909b/pylibsrtp-0.10.0-cp38-abi3-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b2ef1c32d1145239dd0fe7b7fbe083334d345df6b4597fc66faf914a32682d9", size = 2134898 }, + { url = "https://files.pythonhosted.org/packages/00/f6/c76fa5401f9d95c14db70de0cf4fad922ad61686843bc3e7411178a64bc8/pylibsrtp-0.10.0-cp38-abi3-win32.whl", hash = "sha256:8c6fe2576b2ab13942b47db6c2ffe71f5eb1edc1dc3bdd7283169fecd5249e74", size = 1130881 }, + { url = "https://files.pythonhosted.org/packages/4c/31/85a58625edc0b6967fe0904c9d89d019bcece3f3e3bf775b9151a8cf9d0d/pylibsrtp-0.10.0-cp38-abi3-win_amd64.whl", hash = "sha256:cd965d4b0e9a77b362526cab119f4d9ce39b83f1f20f46c6af8e694b86fa19a7", size = 1448840 }, + { url = "https://files.pythonhosted.org/packages/66/b5/30b57cac6adf93dfee20cceba6cd91e216c81b723df2bc9dcfe781456263/pylibsrtp-0.10.0-pp310-pypy310_pp73-macosx_10_9_x86_64.whl", hash = "sha256:582e9771be7ffd060faea215cb4248afdad1356da473df1b8f35c7e382ca3871", size = 1699981 }, + { url = "https://files.pythonhosted.org/packages/16/e8/3846ac56ae4a2de91e9b3e67dff5363b2b07148616d283416fd8dd8c6ca6/pylibsrtp-0.10.0-pp310-pypy310_pp73-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:70111eeb87e5d3ffb9623e1ea036329dc81fed1282aa93c1f32377862ca0a0d8", size = 2441012 }, + { url = "https://files.pythonhosted.org/packages/b1/9f/c611fc47ef5d84dfffca0292bcfb2d78ee5fc1a98d50cf22dfcda3eee171/pylibsrtp-0.10.0-pp310-pypy310_pp73-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eda06947ab42fd3737f01a7b98537a5d5908434d37c70488d10e7bd2ff0d520c", size = 2019497 }, + { url = "https://files.pythonhosted.org/packages/d8/38/90c897fc2f2929290ada1032fa3e0bd39eca9190503250f6724a7bc22b5b/pylibsrtp-0.10.0-pp310-pypy310_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:511158499309c3f7e97e1ebeffbf3dd939e641ea553de43cfc02d3576aad5c15", size = 2074919 }, + { url = "https://files.pythonhosted.org/packages/2c/46/e92f8a8d7cb5c1d68ec85254a8535aad922efa15646c7ba0c7746b42c4ea/pylibsrtp-0.10.0-pp310-pypy310_pp73-win_amd64.whl", hash = "sha256:4033481f332331bf14b9705dca69efd09d3809ba4a2ff69914c53dddf39c20c1", size = 1446426 }, +] + [[package]] name = "pymeta3" version = "0.5.1" @@ -3968,6 +4088,19 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7b/36/88d8438699ba09b714dece00a4a7462330c1d316f5eaa28db450572236f6/pymongo-4.9.2-cp313-cp313-win_amd64.whl", hash = "sha256:169b85728cc17800344ba17d736375f400ef47c9fbb4c42910c4b3e7c0247382", size = 975113 }, ] +[[package]] +name = "pyopenssl" +version = "25.0.0" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cryptography", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "typing-extensions", marker = "(python_full_version < '3.13' and sys_platform == 'darwin') or (python_full_version < '3.13' and sys_platform == 'linux') or (python_full_version < '3.13' and sys_platform == 'win32')" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/9f/26/e25b4a374b4639e0c235527bbe31c0524f26eda701d79456a7e1877f4cc5/pyopenssl-25.0.0.tar.gz", hash = "sha256:cd2cef799efa3936bb08e8ccb9433a575722b9dd986023f1cabc4ae64e9dac16", size = 179573 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/ca/d7/eb76863d2060dcbe7c7e6cccfd95ac02ea0b9acc37745a0d99ff6457aefb/pyOpenSSL-25.0.0-py3-none-any.whl", hash = "sha256:424c247065e46e76a37411b9ab1782541c23bb658bf003772c3405fbaa128e90", size = 56453 }, +] + [[package]] name = "pyparsing" version = "3.2.1" @@ -4804,6 +4937,7 @@ onnx = [ { name = "onnxruntime-genai", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] openai-realtime = [ + { name = "aiortc", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "openai", extra = ["realtime"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] pandas = [ @@ -4850,6 +4984,7 @@ dev = [ [package.metadata] requires-dist = [ { name = "aiohttp", specifier = "~=3.8" }, + { name = "aiortc", marker = "extra == 'openai-realtime'", specifier = ">=1.9.0" }, { name = "anthropic", marker = "extra == 'anthropic'", specifier = "~=0.32" }, { name = "azure-ai-inference", marker = "extra == 'azure'", specifier = ">=1.0.0b6" }, { name = "azure-core-tracing-opentelemetry", marker = "extra == 'azure'", specifier = ">=1.0.0b11" }, From d25a03a73b98f94eb15c2738b9b1d3edf0d85de4 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 15:59:52 +0100 Subject: [PATCH 11/25] added dep --- python/pyproject.toml | 3 ++- python/uv.lock | 17 +++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index a88f332c8cd1..5b9f91d5536b 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -127,7 +127,8 @@ dapr = [ ] openai_realtime = [ "openai[realtime] ~= 1.0", - "aiortc>=1.9.0" + "aiortc>=1.9.0", + "sounddevice>=0.5.1", ] [tool.uv] diff --git a/python/uv.lock b/python/uv.lock index d1d02c8ce8b1..539cd9cfcb1f 100644 --- a/python/uv.lock +++ b/python/uv.lock @@ -4939,6 +4939,7 @@ onnx = [ openai-realtime = [ { name = "aiortc", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, { name = "openai", extra = ["realtime"], marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, + { name = "sounddevice", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, ] pandas = [ { name = "pandas", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, @@ -5031,6 +5032,7 @@ requires-dist = [ { name = "redis", extras = ["hiredis"], marker = "extra == 'redis'", specifier = "~=5.0" }, { name = "redisvl", marker = "extra == 'redis'", specifier = ">=0.3.6" }, { name = "sentence-transformers", marker = "extra == 'hugging-face'", specifier = ">=2.2,<4.0" }, + { name = "sounddevice", marker = "extra == 'openai-realtime'", specifier = ">=0.5.1" }, { name = "taskgroup", marker = "python_full_version < '3.11'", specifier = ">=0.2.2" }, { name = "torch", marker = "extra == 'hugging-face'", specifier = "==2.5.1" }, { name = "transformers", extras = ["torch"], marker = "extra == 'hugging-face'", specifier = "~=4.28" }, @@ -5235,6 +5237,21 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/a5/93/84a16940c44f6ec62cf334f25aed3128a514dffc361397eee09421a1c7f2/snoop-0.6.0-py3-none-any.whl", hash = "sha256:f5ea9060e65594bf404e6841086b4a964cc27bc30569109c91a470f948b0f729", size = 27461 }, ] +[[package]] +name = "sounddevice" +version = "0.5.1" +source = { registry = "https://pypi.org/simple" } +dependencies = [ + { name = "cffi", marker = "sys_platform == 'darwin' or sys_platform == 'linux' or sys_platform == 'win32'" }, +] +sdist = { url = "https://files.pythonhosted.org/packages/80/2d/b04ae180312b81dbb694504bee170eada5372242e186f6298139fd3a0513/sounddevice-0.5.1.tar.gz", hash = "sha256:09ca991daeda8ce4be9ac91e15a9a81c8f81efa6b695a348c9171ea0c16cb041", size = 52896 } +wheels = [ + { url = "https://files.pythonhosted.org/packages/06/d1/464b5fca3decdd0cfec8c47f7b4161a0b12972453201c1bf03811f367c5e/sounddevice-0.5.1-py3-none-any.whl", hash = "sha256:e2017f182888c3f3c280d9fbac92e5dbddac024a7e3442f6e6116bd79dab8a9c", size = 32276 }, + { url = "https://files.pythonhosted.org/packages/6f/f6/6703fe7cf3d7b7279040c792aeec6334e7305956aba4a80f23e62c8fdc44/sounddevice-0.5.1-py3-none-macosx_10_6_x86_64.macosx_10_6_universal2.whl", hash = "sha256:d16cb23d92322526a86a9490c427bf8d49e273d9ccc0bd096feecd229cde6031", size = 107916 }, + { url = "https://files.pythonhosted.org/packages/57/a5/78a5e71f5ec0faedc54f4053775d61407bfbd7d0c18228c7f3d4252fd276/sounddevice-0.5.1-py3-none-win32.whl", hash = "sha256:d84cc6231526e7a08e89beff229c37f762baefe5e0cc2747cbe8e3a565470055", size = 312494 }, + { url = "https://files.pythonhosted.org/packages/af/9b/15217b04f3b36d30de55fef542389d722de63f1ad81f9c72d8afc98cb6ab/sounddevice-0.5.1-py3-none-win_amd64.whl", hash = "sha256:4313b63f2076552b23ac3e0abd3bcfc0c1c6a696fc356759a13bd113c9df90f1", size = 363634 }, +] + [[package]] name = "soupsieve" version = "2.6" From edae188e8eef49dfb23cc341e19a3f9291484197 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 16:02:11 +0100 Subject: [PATCH 12/25] added nd --- python/.cspell.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/.cspell.json b/python/.cspell.json index ea24ad2d7ce4..6abcec319fb0 100644 --- a/python/.cspell.json +++ b/python/.cspell.json @@ -76,6 +76,7 @@ "SEMANTICKERNEL", "OTEL", "vectorizable", - "desync" + "desync", + "nd" ] } \ No newline at end of file From bf4d61df94d007cae46b1497a95b3b5c8f49d236 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 16:04:25 +0100 Subject: [PATCH 13/25] renamed --- python/.cspell.json | 3 +-- python/semantic_kernel/contents/audio_content.py | 4 ++-- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/python/.cspell.json b/python/.cspell.json index 6abcec319fb0..ea24ad2d7ce4 100644 --- a/python/.cspell.json +++ b/python/.cspell.json @@ -76,7 +76,6 @@ "SEMANTICKERNEL", "OTEL", "vectorizable", - "desync", - "nd" + "desync" ] } \ No newline at end of file diff --git a/python/semantic_kernel/contents/audio_content.py b/python/semantic_kernel/contents/audio_content.py index f52b467780b4..b2661ae9ce61 100644 --- a/python/semantic_kernel/contents/audio_content.py +++ b/python/semantic_kernel/contents/audio_content.py @@ -54,6 +54,6 @@ def to_dict(self) -> dict[str, Any]: return {"type": "audio_url", "audio_url": {"uri": str(self)}} @classmethod - def from_nd_array(cls: type[_T], data: ndarray, mime_type: str) -> _T: - """Create an instance from an nd array.""" + def from_ndarray(cls: type[_T], data: ndarray, mime_type: str) -> _T: + """Create an instance from an ndarray.""" return cls(data=data, mime_type=mime_type) From 9427d63b16b7c8112fe27849adcf0118cd14b4e1 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 16:13:42 +0100 Subject: [PATCH 14/25] changed import --- .../ai/open_ai/services/open_ai_realtime_base.py | 3 ++- .../semantic_kernel/connectors/ai/realtime_helpers.py | 10 +++++----- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py index 7caf5a5671df..a4b86218f525 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py @@ -43,7 +43,6 @@ ) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.connectors.ai.realtime_helpers import SKAudioTrack from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.chat_history import ChatHistory from semantic_kernel.contents.function_call_content import FunctionCallContent @@ -682,6 +681,8 @@ async def create_session( **kwargs: Any, ) -> None: """Create a session in the service.""" + from semantic_kernel.connectors.ai.realtime_helpers import SKAudioTrack + ice_servers = [RTCIceServer(urls=["stun:stun.l.google.com:19302"])] self.peer_connection = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) diff --git a/python/semantic_kernel/connectors/ai/realtime_helpers.py b/python/semantic_kernel/connectors/ai/realtime_helpers.py index 94549c402199..b89988f90ab3 100644 --- a/python/semantic_kernel/connectors/ai/realtime_helpers.py +++ b/python/semantic_kernel/connectors/ai/realtime_helpers.py @@ -5,11 +5,11 @@ from typing import Any, Final import numpy as np -import sounddevice as sd from aiortc.mediastreams import MediaStreamError, MediaStreamTrack from av.audio.frame import AudioFrame from av.frame import Frame from pydantic import Field, PrivateAttr +from sounddevice import InputStream, OutputStream from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.kernel_pydantic import KernelBaseModel @@ -37,7 +37,7 @@ class SKAudioTrack(KernelBaseModel, MediaStreamTrack): device: str | int | None = None queue: asyncio.Queue[Frame] = Field(default_factory=asyncio.Queue) is_recording: bool = False - stream: sd.InputStream | None = None + stream: InputStream | None = None frame_size: int = 0 _recording_task: asyncio.Task | None = None _loop: asyncio.AbstractEventLoop | None = None @@ -101,7 +101,7 @@ def callback(indata: np.ndarray, frames: int, time: Any, status: Any) -> None: if self._loop and self._loop.is_running(): asyncio.run_coroutine_threadsafe(self.queue.put(frame), self._loop) - self.stream = sd.InputStream( + self.stream = InputStream( device=self.device, channels=self.channels, samplerate=self.sample_rate, @@ -135,11 +135,11 @@ class SKSimplePlayer(KernelBaseModel): channels: int = PLAYER_CHANNELS frame_duration_ms: int = FRAME_DURATION queue: asyncio.Queue[np.ndarray] = Field(default_factory=asyncio.Queue) - _stream: sd.OutputStream | None = PrivateAttr(None) + _stream: OutputStream | None = PrivateAttr(None) def model_post_init(self, __context: Any) -> None: """Initialize the audio stream.""" - self._stream = sd.OutputStream( + self._stream = OutputStream( callback=self.callback, samplerate=self.sample_rate, channels=self.channels, From 26b17d4a1ed9d80888ada037188626a9a8cea63a Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 17 Jan 2025 16:56:00 +0100 Subject: [PATCH 15/25] binary content fix --- python/semantic_kernel/contents/binary_content.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/semantic_kernel/contents/binary_content.py b/python/semantic_kernel/contents/binary_content.py index 53a4fa5e2bde..1eaa5e30217b 100644 --- a/python/semantic_kernel/contents/binary_content.py +++ b/python/semantic_kernel/contents/binary_content.py @@ -76,7 +76,9 @@ def __init__( kwargs["metadata"].update(data_uri_.parameters) else: kwargs["metadata"] = data_uri_.parameters - elif data is not None: + elif isinstance(data, ndarray): + data_uri_ = DataUri(data_array=data, mime_type=mime_type or self.default_mime_type) + elif data: match data: case str(): data_uri_ = DataUri( @@ -86,8 +88,6 @@ def __init__( data_uri_ = DataUri( data_bytes=data, data_format=data_format, mime_type=mime_type or self.default_mime_type ) - case ndarray(): - data_uri_ = DataUri(data_array=data, mime_type=mime_type or self.default_mime_type) if uri is not None: if isinstance(uri, str) and os.path.exists(uri): From 0eaa5746e5bd6c6a951f715b0d2172c4c6ac6e0d Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 20 Jan 2025 16:31:12 +0100 Subject: [PATCH 16/25] restructured --- python/.cspell.json | 3 +- .../audio/04-chat_with_realtime_api.py | 7 +- .../connectors/ai/open_ai/__init__.py | 3 +- .../open_ai/services/open_ai_model_types.py | 1 + .../ai/open_ai/services/open_ai_realtime.py | 102 +-- .../open_ai/services/open_ai_realtime_base.py | 829 ------------------ .../ai/open_ai/services/realtime/__init__.py | 0 .../ai/open_ai/services/realtime/const.py | 54 ++ .../realtime/open_ai_realtime_base.py | 202 +++++ .../realtime/open_ai_realtime_webrtc.py | 307 +++++++ .../realtime/open_ai_realtime_websocket.py | 201 +++++ .../utils.py} | 0 .../connectors/ai/realtime_client_base.py | 87 +- .../semantic_kernel/contents/audio_content.py | 29 +- 14 files changed, 888 insertions(+), 937 deletions(-) delete mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/realtime/__init__.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/realtime/const.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py create mode 100644 python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py rename python/semantic_kernel/connectors/ai/open_ai/services/{open_ai_realtime_utils.py => realtime/utils.py} (100%) diff --git a/python/.cspell.json b/python/.cspell.json index ea24ad2d7ce4..5a7e0eba650d 100644 --- a/python/.cspell.json +++ b/python/.cspell.json @@ -76,6 +76,7 @@ "SEMANTICKERNEL", "OTEL", "vectorizable", - "desync" + "desync", + "webrtc" ] } \ No newline at end of file diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index 902ad72d48d4..af4024e12849 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -9,11 +9,11 @@ from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai import ( + OpenAIRealtime, OpenAIRealtimeExecutionSettings, - OpenAIRealtimeWebRTC, TurnDetection, ) -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_base import ListenEvents +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_websocket import ListenEvents from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.connectors.ai.realtime_helpers import SKSimplePlayer from semantic_kernel.contents import ChatHistory @@ -26,6 +26,7 @@ aioice_log = logging.getLogger("aioice") aioice_log.setLevel(logging.WARNING) logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) # This simple sample demonstrates how to use the OpenAI Realtime API to create # a chat bot that can listen and respond directly through audio. @@ -119,7 +120,7 @@ async def main() -> None: # create the realtime client and optionally add the audio output function, this is optional audio_player = SKSimplePlayer() - realtime_client = OpenAIRealtimeWebRTC(audio_output=audio_player.realtime_client_callback) + realtime_client = OpenAIRealtime(protocol="webrtc", audio_output=audio_player.realtime_client_callback) # create stream receiver, this can play the audio, if the audio_player is passed # and allows you to print the transcript of the conversation diff --git a/python/semantic_kernel/connectors/ai/open_ai/__init__.py b/python/semantic_kernel/connectors/ai/open_ai/__init__.py index 2c2a87a64a7b..27d36ea30d34 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/__init__.py +++ b/python/semantic_kernel/connectors/ai/open_ai/__init__.py @@ -40,7 +40,7 @@ from semantic_kernel.connectors.ai.open_ai.services.azure_text_to_image import AzureTextToImage from semantic_kernel.connectors.ai.open_ai.services.open_ai_audio_to_text import OpenAIAudioToText from semantic_kernel.connectors.ai.open_ai.services.open_ai_chat_completion import OpenAIChatCompletion -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime, OpenAIRealtimeWebRTC +from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime import OpenAIRealtime from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_completion import OpenAITextCompletion from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding import OpenAITextEmbedding from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio import OpenAITextToAudio @@ -76,7 +76,6 @@ "OpenAIPromptExecutionSettings", "OpenAIRealtime", "OpenAIRealtimeExecutionSettings", - "OpenAIRealtimeWebRTC", "OpenAISettings", "OpenAITextCompletion", "OpenAITextEmbedding", diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_model_types.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_model_types.py index 7a1f43da234e..ea2e05deead7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_model_types.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_model_types.py @@ -12,3 +12,4 @@ class OpenAIModelTypes(Enum): TEXT_TO_IMAGE = "text-to-image" AUDIO_TO_TEXT = "audio-to-text" TEXT_TO_AUDIO = "text-to-audio" + REALTIME = "realtime" diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index 412d0814feb8..9ba373cce6ff 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,80 +1,37 @@ # Copyright (c) Microsoft. All rights reserved. +from ast import TypeVar from collections.abc import Mapping -from typing import Any +from typing import Any, ClassVar, Literal from openai import AsyncOpenAI from pydantic import ValidationError from semantic_kernel.connectors.ai.open_ai.services.open_ai_config_base import OpenAIConfigBase -from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIModelTypes -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_base import ( - OpenAIRealtimeBase, - OpenAIRealtimeWebRTCBase, +from semantic_kernel.connectors.ai.open_ai.services.open_ai_model_types import OpenAIModelTypes +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_webrtc import OpenAIRealtimeWebRTCBase +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_websocket import ( + OpenAIRealtimeWebsocketBase, ) from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError +_T = TypeVar("_T", bound="OpenAIRealtime") -class OpenAIRealtime(OpenAIRealtimeBase, OpenAIConfigBase): - """OpenAI Realtime service.""" - - def __init__( - self, - ai_model_id: str | None = None, - api_key: str | None = None, - org_id: str | None = None, - service_id: str | None = None, - default_headers: Mapping[str, str] | None = None, - async_client: AsyncOpenAI | None = None, - env_file_path: str | None = None, - env_file_encoding: str | None = None, - ) -> None: - """Initialize an OpenAITextCompletion service. - - Args: - ai_model_id (str | None): OpenAI model name, see - https://platform.openai.com/docs/models - service_id (str | None): Service ID tied to the execution settings. - api_key (str | None): The optional API key to use. If provided will override, - the env vars or .env file value. - org_id (str | None): The optional org ID to use. If provided will override, - the env vars or .env file value. - default_headers: The default headers mapping of string keys to - string values for HTTP requests. (Optional) - async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional) - env_file_path (str | None): Use the environment settings file as a fallback to - environment variables. (Optional) - env_file_encoding (str | None): The encoding of the environment settings file. (Optional) - """ - try: - openai_settings = OpenAISettings.create( - api_key=api_key, - org_id=org_id, - text_model_id=ai_model_id, - env_file_path=env_file_path, - env_file_encoding=env_file_encoding, - ) - except ValidationError as ex: - raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex - if not openai_settings.text_model_id: - raise ServiceInitializationError("The OpenAI text model ID is required.") - super().__init__( - ai_model_id=openai_settings.text_model_id, - service_id=service_id, - api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None, - org_id=openai_settings.org_id, - ai_model_type=OpenAIModelTypes.TEXT, - default_headers=default_headers, - client=async_client, - ) - -class OpenAIRealtimeWebRTC(OpenAIRealtimeWebRTCBase, OpenAIConfigBase): +class OpenAIRealtime(OpenAIConfigBase, OpenAIRealtimeBase): """OpenAI Realtime service.""" + def __new__(cls: type["_T"], *args: Any, **kwargs: Any) -> "_T": + """Pick the right subclass, based on protocol.""" + subclass_map = {subcl.protocol: subcl for subcl in cls.__subclasses__()} + subclass = subclass_map[kwargs.pop("protocol", "websocket")] + return super(OpenAIRealtime, subclass).__new__(subclass) + def __init__( self, + protocol: Literal["websocket", "webrtc"] = "websocket", ai_model_id: str | None = None, api_key: str | None = None, org_id: str | None = None, @@ -85,9 +42,10 @@ def __init__( env_file_encoding: str | None = None, **kwargs: Any, ) -> None: - """Initialize an OpenAITextCompletion service. + """Initialize an OpenAIRealtime service. Args: + protocol: The protocol to use, can be either "websocket" or "webrtc". ai_model_id (str | None): OpenAI model name, see https://platform.openai.com/docs/models service_id (str | None): Service ID tied to the execution settings. @@ -116,12 +74,32 @@ def __init__( if not openai_settings.realtime_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") super().__init__( + protocol=protocol, ai_model_id=openai_settings.realtime_model_id, service_id=service_id, api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None, org_id=openai_settings.org_id, - ai_model_type=OpenAIModelTypes.TEXT, + ai_model_type=OpenAIModelTypes.REALTIME, default_headers=default_headers, client=async_client, - **kwargs, ) + + +class OpenAIRealtimeWebRTC(OpenAIRealtime, OpenAIRealtimeWebRTCBase): + """OpenAI Realtime service using WebRTC protocol. + + This should not be used directly, use OpenAIRealtime instead. + Set protocol="webrtc" to use this class. + """ + + protocol: ClassVar[Literal["webrtc"]] = "webrtc" + + +class OpenAIRealtimeWebSocket(OpenAIRealtime, OpenAIRealtimeWebsocketBase): + """OpenAI Realtime service using WebSocket protocol. + + This should not be used directly, use OpenAIRealtime instead. + Set protocol="websocket" to use this class. + """ + + protocol: ClassVar[Literal["websocket"]] = "websocket" diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py deleted file mode 100644 index a4b86218f525..000000000000 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_base.py +++ /dev/null @@ -1,829 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -import asyncio -import base64 -import contextlib -import json -import logging -import sys -from collections.abc import AsyncGenerator, Callable, Coroutine -from enum import Enum -from inspect import isawaitable -from typing import Any, ClassVar, Protocol, cast, runtime_checkable - -if sys.version_info >= (3, 12): - from typing import override # pragma: no cover -else: - from typing_extensions import override # pragma: no cover - -from aiohttp import ClientSession -from aiortc import ( - MediaStreamTrack, - RTCConfiguration, - RTCDataChannel, - RTCIceServer, - RTCPeerConnection, - RTCSessionDescription, -) -from av import AudioFrame -from openai._models import construct_type_unchecked -from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection -from openai.types.beta.realtime.conversation_item_create_event_param import ConversationItemParam -from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent -from pydantic import Field, PrivateAttr - -from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration -from semantic_kernel.connectors.ai.function_calling_utils import ( - prepare_settings_for_function_calling, -) -from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType -from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler -from semantic_kernel.connectors.ai.open_ai.services.open_ai_realtime_utils import ( - update_settings_from_function_call_configuration, -) -from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings -from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.chat_history import ChatHistory -from semantic_kernel.contents.function_call_content import FunctionCallContent -from semantic_kernel.contents.function_result_content import FunctionResultContent -from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent -from semantic_kernel.contents.streaming_text_content import StreamingTextContent -from semantic_kernel.contents.text_content import TextContent -from semantic_kernel.contents.utils.author_role import AuthorRole -from semantic_kernel.kernel import Kernel -from semantic_kernel.utils.experimental_decorator import experimental_class - -logger: logging.Logger = logging.getLogger(__name__) - -# region Protocols - - -@runtime_checkable -@experimental_class -class EventCallBackProtocolAsync(Protocol): - """Event callback protocol.""" - - async def __call__( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool] | None: - """Call the event callback.""" - ... - - -@runtime_checkable -@experimental_class -class EventCallBackProtocol(Protocol): - """Event callback protocol.""" - - def __call__( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool] | None: - """Call the event callback.""" - ... - - -# region Events - - -@experimental_class -class SendEvents(str, Enum): - """Events that can be sent.""" - - SESSION_UPDATE = "session.update" - INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append" - INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit" - INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear" - CONVERSATION_ITEM_CREATE = "conversation.item.create" - CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate" - CONVERSATION_ITEM_DELETE = "conversation.item.delete" - RESPONSE_CREATE = "response.create" - RESPONSE_CANCEL = "response.cancel" - - -@experimental_class -class ListenEvents(str, Enum): - """Events that can be listened to.""" - - ERROR = "error" - SESSION_CREATED = "session.created" - SESSION_UPDATED = "session.updated" - CONVERSATION_CREATED = "conversation.created" - INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed" - INPUT_AUDIO_BUFFER_CLEARED = "input_audio_buffer.cleared" - INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started" - INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped" - CONVERSATION_ITEM_CREATED = "conversation.item.created" - CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_COMPLETED = "conversation.item.input_audio_transcription.completed" - CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_FAILED = "conversation.item.input_audio_transcription.failed" - CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated" - CONVERSATION_ITEM_DELETED = "conversation.item.deleted" - RESPONSE_CREATED = "response.created" - RESPONSE_DONE = "response.done" # contains usage info -> log - RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added" - RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done" - RESPONSE_CONTENT_PART_ADDED = "response.content_part.added" - RESPONSE_CONTENT_PART_DONE = "response.content_part.done" - RESPONSE_TEXT_DELTA = "response.text.delta" - RESPONSE_TEXT_DONE = "response.text.done" - RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.audio_transcript.delta" - RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.audio_transcript.done" - RESPONSE_AUDIO_DELTA = "response.audio.delta" - RESPONSE_AUDIO_DONE = "response.audio.done" - RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" - RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" - RATE_LIMITS_UPDATED = "rate_limits.updated" - - -# region Websocket - - -@experimental_class -class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): - """OpenAI Realtime service.""" - - SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True - connection: AsyncRealtimeConnection | None = None - connected: asyncio.Event = Field(default_factory=asyncio.Event) - event_log: dict[str, list[RealtimeServerEvent]] = Field(default_factory=dict) - event_handlers: dict[str, list[EventCallBackProtocol | EventCallBackProtocolAsync]] = Field(default_factory=dict) - - def model_post_init(self, *args, **kwargs) -> None: - """Post init method for the model.""" - # Register the default event handlers - self.register_event_handler(ListenEvents.RESPONSE_AUDIO_DELTA, self.response_audio_delta_callback) - self.register_event_handler( - ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA, self.response_audio_transcript_delta_callback - ) - self.register_event_handler( - ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DONE, self.response_audio_transcript_done_callback - ) - self.register_event_handler( - ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, self.response_function_call_arguments_delta_callback - ) - self.register_event_handler(ListenEvents.ERROR, self.error_callback) - self.register_event_handler(ListenEvents.SESSION_CREATED, self.session_callback) - self.register_event_handler(ListenEvents.SESSION_UPDATED, self.session_callback) - - def register_event_handler( - self, event_type: str | ListenEvents, handler: EventCallBackProtocol | EventCallBackProtocolAsync - ) -> None: - """Register a event handler.""" - if not isinstance(event_type, ListenEvents): - event_type = ListenEvents(event_type) - self.event_handlers.setdefault(event_type, []).append(handler) - - @override - async def start_listening( - self, - settings: "PromptExecutionSettings", - chat_history: "ChatHistory | None" = None, - **kwargs: Any, - ) -> AsyncGenerator[StreamingChatMessageContent, Any]: - await self.connected.wait() - if not self.connection: - raise ValueError("Connection is not established.") - if not chat_history: - chat_history = ChatHistory() - async for event in self.connection: - event_type = ListenEvents(event.type) - self.event_log.setdefault(event_type, []).append(event) - for handler in self.event_handlers.get(event_type, []): - task = handler(event=event, settings=settings) - if not task: - continue - if isawaitable(task): - async_result = await task - if not async_result: - continue - result, should_return = async_result - else: - result, should_return = task - if should_return: - yield result - else: - chat_history.add_message(result) - - for event_type in self.event_log: - logger.debug(f"Event type: {event_type}, count: {len(self.event_log[event_type])}") - - @override - async def start_sending(self, event: str | SendEvents, **kwargs: Any) -> None: - await self.connected.wait() - if not self.connection: - raise ValueError("Connection is not established.") - if not isinstance(event, SendEvents): - event = SendEvents(event) - match event: - case SendEvents.SESSION_UPDATE: - if "settings" not in kwargs: - logger.error("Event data does not contain 'settings'") - await self.connection.session.update(session=kwargs["settings"].prepare_settings_dict()) - case SendEvents.INPUT_AUDIO_BUFFER_APPEND: - if "content" not in kwargs: - logger.error("Event data does not contain 'content'") - return - await self.connection.input_audio_buffer.append(audio=kwargs["content"].data.decode("utf-8")) - case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: - await self.connection.input_audio_buffer.commit() - case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: - await self.connection.input_audio_buffer.clear() - case SendEvents.CONVERSATION_ITEM_CREATE: - if "item" not in kwargs: - logger.error("Event data does not contain 'item'") - return - content = kwargs["item"] - for item in content.items: - match item: - case TextContent(): - await self.connection.conversation.item.create( - item=ConversationItemParam( - type="message", - content=[ - { - "type": "input_text", - "text": item.text, - } - ], - role="user", - ) - ) - case FunctionCallContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function call needs to have a call_id") - continue - await self.connection.conversation.item.create( - item=ConversationItemParam( - type="function_call", - name=item.name, - arguments=item.arguments, - call_id=call_id, - ) - ) - case FunctionResultContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function result needs to have a call_id") - continue - await self.connection.conversation.item.create( - item=ConversationItemParam( - type="function_call_output", - output=item.result, - call_id=call_id, - ) - ) - case SendEvents.CONVERSATION_ITEM_TRUNCATE: - if "item_id" not in kwargs: - logger.error("Event data does not contain 'item_id'") - return - await self.connection.conversation.item.truncate( - item_id=kwargs["item_id"], content_index=0, audio_end_ms=kwargs.get("audio_end_ms", 0) - ) - case SendEvents.CONVERSATION_ITEM_DELETE: - if "item_id" not in kwargs: - logger.error("Event data does not contain 'item_id'") - return - await self.connection.conversation.item.delete(item_id=kwargs["item_id"]) - case SendEvents.RESPONSE_CREATE: - if "response" in kwargs: - await self.connection.response.create(response=kwargs["response"]) - else: - await self.connection.response.create() - case SendEvents.RESPONSE_CANCEL: - if "response_id" in kwargs: - await self.connection.response.cancel(response_id=kwargs["response_id"]) - else: - await self.connection.response.cancel() - - @override - async def create_session( - self, - settings: PromptExecutionSettings | None = None, - chat_history: ChatHistory | None = None, - **kwargs: Any, - ) -> None: - """Create a session in the service.""" - self.connection = await self.client.beta.realtime.connect(model=self.ai_model_id).enter() - self.connected.set() - if settings or chat_history or kwargs: - await self.update_session(settings=settings, chat_history=chat_history, **kwargs) - - @override - async def update_session( - self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any - ) -> None: - if settings: - if "kernel" in kwargs: - settings = prepare_settings_for_function_calling( - settings, - self.get_prompt_execution_settings_class(), - self._update_function_choice_settings_callback(), - kernel=kwargs.get("kernel"), # type: ignore - ) - await self.start_sending(SendEvents.SESSION_UPDATE, settings=settings) - if chat_history and len(chat_history) > 0: - await asyncio.gather( - *(self.start_sending(SendEvents.CONVERSATION_ITEM_CREATE, item=msg) for msg in chat_history.messages) - ) - - @override - async def close_session(self) -> None: - """Close the session in the service.""" - if self.connected.is_set(): - await self.connection.close() - self.connection = None - self.connected.clear() - - # region Event callbacks - - def response_audio_delta_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response audio delta.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[AudioContent(data=base64.b64decode(event.delta), data_format="base64")], - choice_index=event.content_index, - inner_content=event, - ), True - - def response_audio_transcript_delta_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response audio transcript delta.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[StreamingTextContent(text=event.delta, choice_index=event.content_index)], - choice_index=event.content_index, - inner_content=event, - ), True - - def response_audio_transcript_done_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response audio transcript done.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[StreamingTextContent(text=event.transcript, choice_index=event.content_index)], - choice_index=event.content_index, - inner_content=event, - ), False - - def response_function_call_arguments_delta_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> tuple[Any, bool]: - """Handle response function call arguments delta.""" - return StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[ - FunctionCallContent( - id=event.item_id, - name=event.call_id, - arguments=event.delta, - index=event.output_index, - metadata={"call_id": event.call_id}, - ) - ], - choice_index=0, - inner_content=event, - ), True - - def error_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> None: - """Handle error.""" - logger.error("Error received: %s", event.error) - - def session_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> None: - """Handle session.""" - logger.debug("Session created or updated, session: %s", event.session) - - async def response_function_call_arguments_done_callback( - self, - event: RealtimeServerEvent, - settings: PromptExecutionSettings | None = None, - **kwargs: Any, - ) -> None: - """Handle response function call done.""" - item = FunctionCallContent( - id=event.item_id, - name=event.call_id, - arguments=event.delta, - index=event.output_index, - metadata={"call_id": event.call_id}, - ) - kernel: Kernel | None = kwargs.get("kernel") - call_id = item.name - function_name = next( - output_item_event.item.name - for output_item_event in self.event_log[ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED] - if output_item_event.item.call_id == call_id - ) - item.plugin_name, item.function_name = function_name.split("-", 1) - if kernel: - chat_history = ChatHistory() - await kernel.invoke_function_call(item, chat_history) - await self.start_sending(SendEvents.CONVERSATION_ITEM_CREATE, item=chat_history.messages[-1]) - # The model doesn't start responding to the tool call automatically, so triggering it here. - await self.start_sending(SendEvents.RESPONSE_CREATE) - return chat_history.messages[-1], False - - @override - def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: - from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa - OpenAIRealtimeExecutionSettings, - ) - - return OpenAIRealtimeExecutionSettings - - -# region WebRTC - - -@experimental_class -class OpenAIRealtimeWebRTCBase(OpenAIHandler, RealtimeClientBase): - """OpenAI WebRTC Realtime service.""" - - SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True - peer_connection: RTCPeerConnection | None = None - data_channel: RTCDataChannel | None = None - audio_output: Callable[[AudioFrame], Coroutine[Any, Any, None] | None] | None = None - kernel: Kernel | None = None - - _current_settings: PromptExecutionSettings | None = PrivateAttr(None) - _call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict) - - @override - async def start_listening( - self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, - **kwargs: Any, - ) -> None: - pass - - async def _on_track(self, track: MediaStreamTrack) -> None: - logger.info(f"Received {track.kind} track from remote") - if track.kind != "audio": - return - while True: - try: - # This is a MediaStreamTrack, so the type is AudioFrame - # this might need to be updated if video becomes part of this - frame: AudioFrame = await track.recv() # type: ignore - except Exception as e: - logger.error(f"Error getting audio frame: {e!s}") - break - - try: - if self.audio_output: - out = self.audio_output(frame) - if isawaitable(out): - await out - - except Exception as e: - logger.error(f"Error playing remote audio frame: {e!s}") - try: - await self.receive_buffer.put( - ( - ListenEvents.RESPONSE_AUDIO_DELTA, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame)], # type: ignore - choice_index=0, - ), - ), - ) - except Exception as e: - logger.error(f"Error processing remote audio frame: {e!s}") - await asyncio.sleep(0.01) - - async def _on_data(self, data: str) -> None: - """This method is called whenever a data channel message is received. - - The data is parsed into a RealtimeServerEvent (by OpenAI code) and then processed. - """ - try: - event = cast( - RealtimeServerEvent, - construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)), - ) - except Exception as e: - logger.error(f"Failed to parse event {data} with error: {e!s}") - return - match event.type: - case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA: - await self.receive_buffer.put(( - event.type, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - content=event.delta, - choice_index=event.content_index, - inner_content=event, - ), - )) - case ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED: - if event.item.type == "function_call": - self._call_id_to_function_map[event.item.call_id] = event.item.name - case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA: - await self.receive_buffer.put(( - event.type, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[ - FunctionCallContent( - id=event.item_id, - name=event.call_id, - arguments=event.delta, - index=event.output_index, - metadata={"call_id": event.call_id}, - ) - ], - choice_index=0, - inner_content=event, - ), - )) - case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE: - await self._handle_function_call_arguments_done(event) - case ListenEvents.ERROR: - logger.error("Error received: %s", event.error) - case ListenEvents.SESSION_CREATED, ListenEvents.SESSION_UPDATED: - logger.info("Session created or updated, session: %s", event.session) - case _: - logger.debug(f"Received event: {event}") - # we put all event in the output buffer, but after the interpreted one. - # so when dealing with them, make sure to check the type of the event, since they - # might be of different types. - await self.receive_buffer.put((event.type, event)) - - @override - async def start_sending(self, **kwargs: Any) -> None: - while True: - item = await self.send_buffer.get() - if not item: - continue - if isinstance(item, tuple): - event, data = item - else: - event = item - data = {} - if not isinstance(event, SendEvents): - event = SendEvents(event) - response: dict[str, Any] = {"type": event.value} - match event: - case SendEvents.SESSION_UPDATE: - if "settings" not in data: - logger.error("Event data does not contain 'settings'") - response["session"] = data["settings"].prepare_settings_dict() - case SendEvents.CONVERSATION_ITEM_CREATE: - if "item" not in data: - logger.error("Event data does not contain 'item'") - return - content = data["item"] - for item in content.items: - match item: - case TextContent(): - response["item"] = ConversationItemParam( - type="message", - content=[ - { - "type": "input_text", - "text": item.text, - } - ], - role="user", - ) - - case FunctionCallContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function call needs to have a call_id") - continue - response["item"] = ConversationItemParam( - type="function_call", - name=item.name, - arguments=item.arguments, - call_id=call_id, - ) - - case FunctionResultContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function result needs to have a call_id") - continue - response["item"] = ConversationItemParam( - type="function_call_output", - output=item.result, - call_id=call_id, - ) - - case SendEvents.CONVERSATION_ITEM_TRUNCATE: - if "item_id" not in data: - logger.error("Event data does not contain 'item_id'") - return - response["item_id"] = data["item_id"] - response["content_index"] = 0 - response["audio_end_ms"] = data.get("audio_end_ms", 0) - - case SendEvents.CONVERSATION_ITEM_DELETE: - if "item_id" not in data: - logger.error("Event data does not contain 'item_id'") - return - response["item_id"] = data["item_id"] - case SendEvents.RESPONSE_CREATE: - if "response" in data: - response["response"] = data["response"] - case SendEvents.RESPONSE_CANCEL: - if "response_id" in data: - response["response_id"] = data["response_id"] - - if self.data_channel: - while self.data_channel.readyState != "open": - await asyncio.sleep(0.1) - try: - self.data_channel.send(json.dumps(response)) - except Exception as e: - logger.error(f"Failed to send event {event} with error: {e!s}") - - @override - async def create_session( - self, - settings: PromptExecutionSettings | None = None, - chat_history: ChatHistory | None = None, - audio_track: MediaStreamTrack | None = None, - **kwargs: Any, - ) -> None: - """Create a session in the service.""" - from semantic_kernel.connectors.ai.realtime_helpers import SKAudioTrack - - ice_servers = [RTCIceServer(urls=["stun:stun.l.google.com:19302"])] - self.peer_connection = RTCPeerConnection(configuration=RTCConfiguration(iceServers=ice_servers)) - - self.peer_connection.on("track")(self._on_track) - - self.data_channel = self.peer_connection.createDataChannel("oai-events", protocol="json") - self.data_channel.on("message")(self._on_data) - - self.peer_connection.addTransceiver(audio_track or SKAudioTrack(), "sendrecv") - - offer = await self.peer_connection.createOffer() - await self.peer_connection.setLocalDescription(offer) - - try: - ephemeral_token = await self.get_ephemeral_token() - headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"} - - async with ( - ClientSession() as session, - session.post( - f"{self.client.beta.realtime._client.base_url}realtime?model={self.ai_model_id}", - headers=headers, - data=offer.sdp, - ) as response, - ): - if response.status not in [200, 201]: - error_text = await response.text() - raise Exception(f"OpenAI WebRTC error: {error_text}") - - sdp_answer = await response.text() - answer = RTCSessionDescription(sdp=sdp_answer, type="answer") - await self.peer_connection.setRemoteDescription(answer) - logger.info("Connected to OpenAI WebRTC") - - except Exception as e: - logger.error(f"Failed to connect to OpenAI: {e!s}") - raise - - if settings or chat_history or kwargs: - await self.update_session(settings=settings, chat_history=chat_history, **kwargs) - - @override - async def update_session( - self, - settings: PromptExecutionSettings | None = None, - chat_history: ChatHistory | None = None, - create_response: bool = True, - **kwargs: Any, - ) -> None: - if "kernel" in kwargs: - self.kernel = kwargs["kernel"] - if settings: - self._current_settings = settings - if self._current_settings and self.kernel: - self._current_settings = prepare_settings_for_function_calling( - self._current_settings, - self.get_prompt_execution_settings_class(), - self._update_function_choice_settings_callback(), - kernel=self.kernel, # type: ignore - ) - await self.send_buffer.put((SendEvents.SESSION_UPDATE, {"settings": self._current_settings})) - if chat_history and len(chat_history) > 0: - for msg in chat_history.messages: - await self.send_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": msg})) - if create_response: - await self.send_buffer.put(SendEvents.RESPONSE_CREATE) - - @override - async def close_session(self) -> None: - """Close the session in the service.""" - if self.peer_connection: - with contextlib.suppress(asyncio.CancelledError): - await self.peer_connection.close() - self.peer_connection = None - if self.data_channel: - with contextlib.suppress(asyncio.CancelledError): - self.data_channel.close() - self.data_channel = None - - async def _handle_function_call_arguments_done( - self, - event: RealtimeServerEvent, - ) -> None: - """Handle response function call done.""" - plugin_name, function_name = self._call_id_to_function_map.pop(event.call_id, "-").split("-", 1) - if not plugin_name or not function_name: - logger.error("Function call needs to have a plugin name and function name") - return - item = FunctionCallContent( - id=event.item_id, - plugin_name=plugin_name, - function_name=function_name, - arguments=event.arguments, - index=event.output_index, - metadata={"call_id": event.call_id}, - ) - if not self.kernel and not self._current_settings.function_choice_behavior.auto_invoke_kernel_functions: - return - chat_history = ChatHistory() - await self.kernel.invoke_function_call(item, chat_history) - created_output = chat_history.messages[-1] - # This returns the output to the service - await self.send_buffer.put((SendEvents.CONVERSATION_ITEM_CREATE, {"item": created_output})) - # The model doesn't start responding to the tool call automatically, so triggering it here. - await self.send_buffer.put(SendEvents.RESPONSE_CREATE) - # This allows a user to have a full conversation in his code - await self.receive_buffer.put((ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, created_output)) - - async def get_ephemeral_token(self) -> str: - """Get an ephemeral token from OpenAI.""" - headers = {"Authorization": f"Bearer {self.client.api_key}", "Content-Type": "application/json"} - data = {"model": self.ai_model_id, "voice": "echo"} - - try: - async with ( - ClientSession() as session, - session.post( - f"{self.client.beta.realtime._client.base_url}/realtime/sessions", headers=headers, json=data - ) as response, - ): - if response.status not in [200, 201]: - error_text = await response.text() - raise Exception(f"Failed to get ephemeral token: {error_text}") - - result = await response.json() - return result["client_secret"]["value"] - - except Exception as e: - logger.error(f"Failed to get ephemeral token: {e!s}") - raise - - @override - def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: - from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa - OpenAIRealtimeExecutionSettings, - ) - - return OpenAIRealtimeExecutionSettings - - @override - def _update_function_choice_settings_callback( - self, - ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: - return update_settings_from_function_call_configuration diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/__init__.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/const.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/const.py new file mode 100644 index 000000000000..533e00d24d53 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/const.py @@ -0,0 +1,54 @@ +# Copyright (c) Microsoft. All rights reserved. + +from enum import Enum + +from semantic_kernel.utils.experimental_decorator import experimental_class + + +@experimental_class +class SendEvents(str, Enum): + """Events that can be sent.""" + + SESSION_UPDATE = "session.update" + INPUT_AUDIO_BUFFER_APPEND = "input_audio_buffer.append" + INPUT_AUDIO_BUFFER_COMMIT = "input_audio_buffer.commit" + INPUT_AUDIO_BUFFER_CLEAR = "input_audio_buffer.clear" + CONVERSATION_ITEM_CREATE = "conversation.item.create" + CONVERSATION_ITEM_TRUNCATE = "conversation.item.truncate" + CONVERSATION_ITEM_DELETE = "conversation.item.delete" + RESPONSE_CREATE = "response.create" + RESPONSE_CANCEL = "response.cancel" + + +@experimental_class +class ListenEvents(str, Enum): + """Events that can be listened to.""" + + ERROR = "error" + SESSION_CREATED = "session.created" + SESSION_UPDATED = "session.updated" + CONVERSATION_CREATED = "conversation.created" + INPUT_AUDIO_BUFFER_COMMITTED = "input_audio_buffer.committed" + INPUT_AUDIO_BUFFER_CLEARED = "input_audio_buffer.cleared" + INPUT_AUDIO_BUFFER_SPEECH_STARTED = "input_audio_buffer.speech_started" + INPUT_AUDIO_BUFFER_SPEECH_STOPPED = "input_audio_buffer.speech_stopped" + CONVERSATION_ITEM_CREATED = "conversation.item.created" + CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_COMPLETED = "conversation.item.input_audio_transcription.completed" + CONVERSATION_ITEM_INPUT_AUDIO_TRANSCRIPTION_FAILED = "conversation.item.input_audio_transcription.failed" + CONVERSATION_ITEM_TRUNCATED = "conversation.item.truncated" + CONVERSATION_ITEM_DELETED = "conversation.item.deleted" + RESPONSE_CREATED = "response.created" + RESPONSE_DONE = "response.done" # contains usage info -> log + RESPONSE_OUTPUT_ITEM_ADDED = "response.output_item.added" + RESPONSE_OUTPUT_ITEM_DONE = "response.output_item.done" + RESPONSE_CONTENT_PART_ADDED = "response.content_part.added" + RESPONSE_CONTENT_PART_DONE = "response.content_part.done" + RESPONSE_TEXT_DELTA = "response.text.delta" + RESPONSE_TEXT_DONE = "response.text.done" + RESPONSE_AUDIO_TRANSCRIPT_DELTA = "response.audio_transcript.delta" + RESPONSE_AUDIO_TRANSCRIPT_DONE = "response.audio_transcript.done" + RESPONSE_AUDIO_DELTA = "response.audio.delta" + RESPONSE_AUDIO_DONE = "response.audio.done" + RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA = "response.function_call_arguments.delta" + RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE = "response.function_call_arguments.done" + RATE_LIMITS_UPDATED = "rate_limits.updated" diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py new file mode 100644 index 000000000000..9f72ee1fd5d1 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py @@ -0,0 +1,202 @@ +# Copyright (c) Microsoft. All rights reserved. + +import logging +import sys +from collections.abc import Callable, Coroutine +from typing import TYPE_CHECKING, Any, ClassVar, Literal + +if sys.version_info >= (3, 12): + from typing import override # pragma: no cover +else: + from typing_extensions import override # pragma: no cover + +from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent +from openai.types.beta.realtime.response_function_call_arguments_done_event import ( + ResponseFunctionCallArgumentsDoneEvent, +) +from pydantic import PrivateAttr + +from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration +from semantic_kernel.connectors.ai.function_calling_utils import ( + prepare_settings_for_function_calling, +) +from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType +from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.connectors.ai.open_ai.services.realtime.utils import ( + update_settings_from_function_call_configuration, +) +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.kernel import Kernel +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings + from semantic_kernel.contents.chat_history import ChatHistory + + +logger: logging.Logger = logging.getLogger(__name__) + + +@experimental_class +class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): + """OpenAI Realtime service.""" + + protocol: ClassVar[Literal["websocket", "webrtc"]] = "websocket" + SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True + audio_output: Callable[[Any], Coroutine[Any, Any, None] | None] | None = None + kernel: Kernel | None = None + + _current_settings: PromptExecutionSettings | None = PrivateAttr(None) + _call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict) + + async def _handle_event(self, event: RealtimeServerEvent) -> None: + """Handle all events but audio delta. + + Audio delta has to be handled by the implementation of the protocol as some + protocols have different ways of handling audio. + + + """ + match event.type: + case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value: + await self.receive_buffer.put(( + event.type, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + content=event.delta, + choice_index=event.content_index, + inner_content=event, + ), + )) + case ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED.value: + if event.item.type == "function_call" and event.item.call_id and event.item.name: + self._call_id_to_function_map[event.item.call_id] = event.item.name + case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA.value: + await self.receive_buffer.put(( + event.type, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + FunctionCallContent( + id=event.item_id, + name=event.call_id, + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + ], + choice_index=0, + inner_content=event, + ), + )) + case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE.value: + await self._handle_function_call_arguments_done(event) + case ListenEvents.ERROR.value: + logger.error("Error received: %s", event.error) + case ListenEvents.SESSION_CREATED.value, ListenEvents.SESSION_UPDATED.value: + logger.info("Session created or updated, session: %s", event.session) + case _: + logger.debug(f"Received event: {event}") + # we put all event in the output buffer, but after the interpreted one. + # so when dealing with them, make sure to check the type of the event, since they + # might be of different types. + await self.receive_buffer.put((event.type, event)) + + @override + async def update_session( + self, + settings: PromptExecutionSettings | None = None, + chat_history: ChatHistory | None = None, + create_response: bool = False, + **kwargs: Any, + ) -> None: + if "kernel" in kwargs: + self.kernel = kwargs["kernel"] + if settings: + self._current_settings = settings + if self._current_settings and self.kernel: + self._current_settings = prepare_settings_for_function_calling( + self._current_settings, + self.get_prompt_execution_settings_class(), + self._update_function_choice_settings_callback(), + kernel=self.kernel, # type: ignore + ) + await self.send(SendEvents.SESSION_UPDATE, settings=self._current_settings) + if chat_history and len(chat_history) > 0: + for msg in chat_history.messages: + await self.send(SendEvents.CONVERSATION_ITEM_CREATE, item=msg) + if create_response: + await self.send(SendEvents.RESPONSE_CREATE) + + @override + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: + from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa + OpenAIRealtimeExecutionSettings, + ) + + return OpenAIRealtimeExecutionSettings + + @override + def _update_function_choice_settings_callback( + self, + ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: + return update_settings_from_function_call_configuration + + async def _handle_function_call_arguments_done( + self, + event: ResponseFunctionCallArgumentsDoneEvent, + ) -> None: + """Handle response function call done.""" + if not self.kernel or ( + self._current_settings + and self._current_settings.function_choice_behavior + and not self._current_settings.function_choice_behavior.auto_invoke_kernel_functions + ): + return + plugin_name, function_name = self._call_id_to_function_map.pop(event.call_id, "-").split("-", 1) + if not plugin_name or not function_name: + logger.error("Function call needs to have a plugin name and function name") + return + item = FunctionCallContent( + id=event.item_id, + plugin_name=plugin_name, + function_name=function_name, + arguments=event.arguments, + index=event.output_index, + metadata={"call_id": event.call_id}, + ) + chat_history = ChatHistory() + await self.kernel.invoke_function_call(item, chat_history) + created_output = chat_history.messages[-1] + # This returns the output to the service + await self.send(SendEvents.CONVERSATION_ITEM_CREATE, item=created_output) + # The model doesn't start responding to the tool call automatically, so triggering it here. + await self.send(SendEvents.RESPONSE_CREATE) + # This allows a user to have a full conversation in his code + await self.receive_buffer.put((ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, created_output)) + + @override + async def start_listening( + self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any + ) -> None: + pass + + @override + async def start_sending(self, **kwargs: Any) -> None: + pass + + @override + async def create_session( + self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any + ) -> None: + pass + + @override + async def close_session(self) -> None: + pass diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py new file mode 100644 index 000000000000..583e34bfd997 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py @@ -0,0 +1,307 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import contextlib +import json +import logging +import sys +from collections.abc import Callable, Coroutine +from inspect import isawaitable +from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast + +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase + +if sys.version_info >= (3, 12): + from typing import override # pragma: no cover +else: + from typing_extensions import override # pragma: no cover + +from aiohttp import ClientSession +from aiortc import ( + RTCConfiguration, + RTCDataChannel, + RTCIceServer, + RTCPeerConnection, + RTCSessionDescription, +) +from av.audio.frame import AudioFrame +from openai._models import construct_type_unchecked +from openai.types.beta.realtime.conversation_item_param import ConversationItemParam +from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent + +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from aiortc import MediaStreamTrack + + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings + from semantic_kernel.contents.chat_history import ChatHistory + + +logger: logging.Logger = logging.getLogger(__name__) + + +@experimental_class +class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase): + """OpenAI WebRTC Realtime service.""" + + protocol: ClassVar[Literal["webrtc"]] = "webrtc" + peer_connection: RTCPeerConnection | None = None + data_channel: RTCDataChannel | None = None + audio_output: Callable[[AudioFrame], Coroutine[Any, Any, None] | None] | None = None + + # region public methods + + @override + async def start_listening( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + create_response: bool = False, + **kwargs: Any, + ) -> None: + if chat_history or settings or create_response: + await self.update_session(settings=settings, chat_history=chat_history, create_response=create_response) + + @override + async def start_sending(self, **kwargs: Any) -> None: + if not self.data_channel: + logger.error("Data channel not initialized") + return + while self.data_channel.readyState != "open": + await asyncio.sleep(0.1) + while True: + event, data = await self.send_buffer.get() + if not isinstance(event, SendEvents): + event = SendEvents(event) + response: dict[str, Any] = {"type": event.value} + match event: + case SendEvents.SESSION_UPDATE: + if "settings" not in data: + logger.error("Event data does not contain 'settings'") + response["session"] = data["settings"].prepare_settings_dict() + case SendEvents.CONVERSATION_ITEM_CREATE: + if "item" not in data: + logger.error("Event data does not contain 'item'") + return + content = data["item"] + for item in content.items: + match item: + case TextContent(): + response["item"] = ConversationItemParam( + type="message", + content=[ + { + "type": "input_text", + "text": item.text, + } + ], + role="user", + ) + + case FunctionCallContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function call needs to have a call_id") + continue + response["item"] = ConversationItemParam( + type="function_call", + name=item.name or item.function_name, + arguments="" + if not item.arguments + else item.arguments + if isinstance(item.arguments, str) + else json.dumps(item.arguments), + call_id=call_id, + ) + + case FunctionResultContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function result needs to have a call_id") + continue + response["item"] = ConversationItemParam( + type="function_call_output", + output=item.result, + call_id=call_id, + ) + + case SendEvents.CONVERSATION_ITEM_TRUNCATE: + if "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + response["item_id"] = data["item_id"] + response["content_index"] = 0 + response["audio_end_ms"] = data.get("audio_end_ms", 0) + + case SendEvents.CONVERSATION_ITEM_DELETE: + if "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + response["item_id"] = data["item_id"] + case SendEvents.RESPONSE_CREATE: + if "response" in data: + response["response"] = data["response"] + case SendEvents.RESPONSE_CANCEL: + if "response_id" in data: + response["response_id"] = data["response_id"] + + try: + self.data_channel.send(json.dumps(response)) + except Exception as e: + logger.error(f"Failed to send event {event} with error: {e!s}") + + @override + async def create_session( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + audio_track: "MediaStreamTrack | None" = None, + **kwargs: Any, + ) -> None: + """Create a session in the service.""" + if not audio_track: + from semantic_kernel.connectors.ai.realtime_helpers import SKAudioTrack + + audio_track = SKAudioTrack() + + self.peer_connection = RTCPeerConnection( + configuration=RTCConfiguration(iceServers=[RTCIceServer(urls="stun:stun.l.google.com:19302")]) + ) + + # track is the audio track being returned from the service + self.peer_connection.on("track")(self._on_track) + + # data channel is used to send and receive messages + self.data_channel = self.peer_connection.createDataChannel("oai-events", protocol="json") + self.data_channel.on("message")(self._on_data) + + # this is the incoming audio, which sends audio to the service + self.peer_connection.addTransceiver(audio_track) + + offer = await self.peer_connection.createOffer() + await self.peer_connection.setLocalDescription(offer) + + try: + ephemeral_token = await self._get_ephemeral_token() + headers = {"Authorization": f"Bearer {ephemeral_token}", "Content-Type": "application/sdp"} + + async with ( + ClientSession() as session, + session.post( + f"{self.client.beta.realtime._client.base_url}realtime?model={self.ai_model_id}", + headers=headers, + data=offer.sdp, + ) as response, + ): + if response.status not in [200, 201]: + error_text = await response.text() + raise Exception(f"OpenAI WebRTC error: {error_text}") + + sdp_answer = await response.text() + answer = RTCSessionDescription(sdp=sdp_answer, type="answer") + await self.peer_connection.setRemoteDescription(answer) + logger.info("Connected to OpenAI WebRTC") + + except Exception as e: + logger.error(f"Failed to connect to OpenAI: {e!s}") + raise + + if settings or chat_history or kwargs: + await self.update_session(settings=settings, chat_history=chat_history, **kwargs) + + @override + async def close_session(self) -> None: + """Close the session in the service.""" + if self.peer_connection: + with contextlib.suppress(asyncio.CancelledError): + await self.peer_connection.close() + self.peer_connection = None + if self.data_channel: + with contextlib.suppress(asyncio.CancelledError): + self.data_channel.close() + self.data_channel = None + + # region implementation specifics + + async def _on_track(self, track: "MediaStreamTrack") -> None: + logger.info(f"Received {track.kind} track from remote") + if track.kind != "audio": + return + while True: + try: + # This is a MediaStreamTrack, so the type is AudioFrame + # this might need to be updated if video becomes part of this + frame: AudioFrame = await track.recv() # type: ignore + except Exception as e: + logger.error(f"Error getting audio frame: {e!s}") + break + + try: + if self.audio_output: + out = self.audio_output(frame) + if isawaitable(out): + await out + + except Exception as e: + logger.error(f"Error playing remote audio frame: {e!s}") + try: + await self.receive_buffer.put( + ( + ListenEvents.RESPONSE_AUDIO_DELTA, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame)], # type: ignore + choice_index=0, + ), + ), + ) + except Exception as e: + logger.error(f"Error processing remote audio frame: {e!s}") + await asyncio.sleep(0.01) + + async def _on_data(self, data: str) -> None: + """This method is called whenever a data channel message is received. + + The data is parsed into a RealtimeServerEvent (by OpenAI code) and then processed. + Audio data is not send through this channel, use _on_track for that. + """ + try: + event = cast( + RealtimeServerEvent, + construct_type_unchecked(value=json.loads(data), type_=cast(Any, RealtimeServerEvent)), + ) + except Exception as e: + logger.error(f"Failed to parse event {data} with error: {e!s}") + return + await self._handle_event(event) + + async def _get_ephemeral_token(self) -> str: + """Get an ephemeral token from OpenAI.""" + headers = {"Authorization": f"Bearer {self.client.api_key}", "Content-Type": "application/json"} + data = {"model": self.ai_model_id, "voice": "echo"} + + try: + async with ( + ClientSession() as session, + session.post( + f"{self.client.beta.realtime._client.base_url}/realtime/sessions", headers=headers, json=data + ) as response, + ): + if response.status not in [200, 201]: + error_text = await response.text() + raise Exception(f"Failed to get ephemeral token: {error_text}") + + result = await response.json() + return result["client_secret"]["value"] + + except Exception as e: + logger.error(f"Failed to get ephemeral token: {e!s}") + raise diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py new file mode 100644 index 000000000000..95ff1ab3a6b8 --- /dev/null +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py @@ -0,0 +1,201 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import base64 +import json +import logging +import sys +from inspect import isawaitable +from typing import TYPE_CHECKING, Any, ClassVar, Literal + +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase + +if sys.version_info >= (3, 12): + from typing import override # pragma: no cover +else: + from typing_extensions import override # pragma: no cover + +from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection +from openai.types.beta.realtime.conversation_item_param import ConversationItemParam +from pydantic import Field + +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.utils.experimental_decorator import experimental_class + +if TYPE_CHECKING: + from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings + from semantic_kernel.contents.chat_history import ChatHistory + +logger: logging.Logger = logging.getLogger(__name__) + +# region Websocket + + +@experimental_class +class OpenAIRealtimeWebsocketBase(OpenAIRealtimeBase): + """OpenAI Realtime service.""" + + protocol: ClassVar[Literal["websocket"]] = "websocket" + connection: AsyncRealtimeConnection | None = None + connected: asyncio.Event = Field(default_factory=asyncio.Event) + + @override + async def start_listening( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + create_response: bool = False, + **kwargs: Any, + ) -> None: + await self.connected.wait() + if not self.connection: + raise ValueError("Connection is not established.") + + if chat_history or settings or create_response: + await self.update_session(settings=settings, chat_history=chat_history, create_response=create_response) + + async for event in self.connection: + if event.type == ListenEvents.RESPONSE_AUDIO_DELTA.value: + if self.audio_output: + out = self.audio_output(event) + if isawaitable(out): + await out + try: + await self.receive_buffer.put(( + event.type, + StreamingChatMessageContent( + role=AuthorRole.ASSISTANT, + items=[ + AudioContent( + data=base64.b64decode(event.delta), + data_format="base64", + inner_content=event, + ) + ], # type: ignore + choice_index=event.content_index, + ), + )) + except Exception as e: + logger.error(f"Error processing remote audio frame: {e!s}") + else: + await self._handle_event(event) + + @override + async def start_sending(self, **kwargs: Any) -> None: + await self.connected.wait() + if not self.connection: + raise ValueError("Connection is not established.") + while True: + event, data = await self.send_buffer.get() + match event: + case SendEvents.SESSION_UPDATE: + if "settings" not in data: + logger.error("Event data does not contain 'settings'") + await self.connection.session.update(session=data["settings"].prepare_settings_dict()) + case SendEvents.INPUT_AUDIO_BUFFER_APPEND: + if "content" not in data: + logger.error("Event data does not contain 'content'") + return + await self.connection.input_audio_buffer.append(audio=data["content"].data.decode("utf-8")) + case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: + await self.connection.input_audio_buffer.commit() + case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: + await self.connection.input_audio_buffer.clear() + case SendEvents.CONVERSATION_ITEM_CREATE: + if "item" not in data: + logger.error("Event data does not contain 'item'") + return + content = data["item"] + for item in content.items: + match item: + case TextContent(): + await self.connection.conversation.item.create( + item=ConversationItemParam( + type="message", + content=[ + { + "type": "input_text", + "text": item.text, + } + ], + role="user", + ) + ) + case FunctionCallContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function call needs to have a call_id") + continue + await self.connection.conversation.item.create( + item=ConversationItemParam( + type="function_call", + name=item.name or item.function_name, + arguments="" + if not item.arguments + else item.arguments + if isinstance(item.arguments, str) + else json.dumps(item.arguments), + call_id=call_id, + ) + ) + case FunctionResultContent(): + call_id = item.metadata.get("call_id") + if not call_id: + logger.error("Function result needs to have a call_id") + continue + await self.connection.conversation.item.create( + item=ConversationItemParam( + type="function_call_output", + output=item.result, + call_id=call_id, + ) + ) + case SendEvents.CONVERSATION_ITEM_TRUNCATE: + if "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + await self.connection.conversation.item.truncate( + item_id=data["item_id"], content_index=0, audio_end_ms=data.get("audio_end_ms", 0) + ) + case SendEvents.CONVERSATION_ITEM_DELETE: + if "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + await self.connection.conversation.item.delete(item_id=data["item_id"]) + case SendEvents.RESPONSE_CREATE: + if "response" in data: + await self.connection.response.create(response=data["response"]) + else: + await self.connection.response.create() + case SendEvents.RESPONSE_CANCEL: + if "response_id" in data: + await self.connection.response.cancel(response_id=data["response_id"]) + else: + await self.connection.response.cancel() + + @override + async def create_session( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + **kwargs: Any, + ) -> None: + """Create a session in the service.""" + self.connection = await self.client.beta.realtime.connect(model=self.ai_model_id).enter() + self.connected.set() + if settings or chat_history or kwargs: + await self.update_session(settings=settings, chat_history=chat_history, **kwargs) + + @override + async def close_session(self) -> None: + """Close the session in the service.""" + if self.connected.is_set() and self.connection: + await self.connection.close() + self.connection = None + self.connected.clear() diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_utils.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py similarity index 100% rename from python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime_utils.py rename to python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index 991854987faa..5f6fa302d545 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -28,52 +28,47 @@ class RealtimeClientBase(AIServiceClientBase, ABC): """Base class for a realtime client.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False - send_buffer: Queue[str | tuple[str, Any]] = Field(default_factory=Queue) + send_buffer: Queue[tuple[str, Any]] = Field(default_factory=Queue) receive_buffer: Queue[tuple[str, Any]] = Field(default_factory=Queue) - async def __aenter__(self) -> "RealtimeClientBase": - """Enter the context manager. + async def send(self, event: str, **kwargs: Any) -> None: + """Send an event to the service. - Default implementation calls the create session method. + Args: + event: The event to send. + kwargs: Additional arguments. """ - await self.create_session() - return self - - async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: - """Exit the context manager.""" - await self.close_session() + await self.send_buffer.put((event, kwargs)) - @abstractmethod - async def close_session(self) -> None: - """Close the session in the service.""" - pass - - @abstractmethod - async def create_session( + async def start_streaming( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> None: - """Create a session in the service. + """Start streaming, will start both listening and sending. + + This method, start tasks for both listening and sending. + + The arguments are passed to the start_listening method. Args: settings: Prompt execution settings. chat_history: Chat history. kwargs: Additional arguments. """ - raise NotImplementedError + async with TaskGroup() as tg: + tg.create_task(self.start_listening(settings=settings, chat_history=chat_history, **kwargs)) + tg.create_task(self.start_sending(**kwargs)) @abstractmethod - async def update_session( + async def start_listening( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> None: - """Update a session in the service. - - Can be used when using the context manager instead of calling create_session with these same arguments. + """Starts listening for messages from the service, adds them to the output_buffer. Args: settings: Prompt execution settings. @@ -82,35 +77,39 @@ async def update_session( """ raise NotImplementedError - async def start_streaming( + @abstractmethod + async def start_sending( + self, + ) -> None: + """Start sending items from the input_buffer to the service.""" + raise NotImplementedError + + @abstractmethod + async def create_session( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> None: - """Start streaming, will start both listening and sending. - - This method, start tasks for both listening and sending. - - The arguments are passed to the start_listening method. + """Create a session in the service. Args: settings: Prompt execution settings. chat_history: Chat history. kwargs: Additional arguments. """ - async with TaskGroup() as tg: - tg.create_task(self.start_listening(settings=settings, chat_history=chat_history, **kwargs)) - tg.create_task(self.start_sending(**kwargs)) + raise NotImplementedError @abstractmethod - async def start_listening( + async def update_session( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> None: - """Starts listening for messages from the service, adds them to the output_buffer. + """Update a session in the service. + + Can be used when using the context manager instead of calling create_session with these same arguments. Args: settings: Prompt execution settings. @@ -120,11 +119,9 @@ async def start_listening( raise NotImplementedError @abstractmethod - async def start_sending( - self, - ) -> None: - """Start sending items from the input_buffer to the service.""" - raise NotImplementedError + async def close_session(self) -> None: + """Close the session in the service.""" + pass def _update_function_choice_settings_callback( self, @@ -135,3 +132,15 @@ def _update_function_choice_settings_callback( update the settings from a function call configuration. """ return lambda configuration, settings, choice_type: None + + async def __aenter__(self) -> "RealtimeClientBase": + """Enter the context manager. + + Default implementation calls the create session method. + """ + await self.create_session() + return self + + async def __aexit__(self, exc_type, exc_val, exc_tb) -> None: + """Exit the context manager.""" + await self.close_session() diff --git a/python/semantic_kernel/contents/audio_content.py b/python/semantic_kernel/contents/audio_content.py index b2661ae9ce61..b7e157a242bf 100644 --- a/python/semantic_kernel/contents/audio_content.py +++ b/python/semantic_kernel/contents/audio_content.py @@ -5,8 +5,9 @@ from numpy import ndarray from pydantic import Field +from pydantic_core import Url -from semantic_kernel.contents.binary_content import BinaryContent +from semantic_kernel.contents.binary_content import BinaryContent, DataUrl from semantic_kernel.contents.const import AUDIO_CONTENT_TAG, ContentTypes from semantic_kernel.utils.experimental_decorator import experimental_class @@ -42,6 +43,32 @@ class AudioContent(BinaryContent): content_type: Literal[ContentTypes.AUDIO_CONTENT] = Field(AUDIO_CONTENT_TAG, init=False) # type: ignore tag: ClassVar[str] = AUDIO_CONTENT_TAG + def __init__( + self, + uri: Url | str | None = None, + data_uri: DataUrl | str | None = None, + data: str | bytes | ndarray | None = None, + data_format: str | None = None, + mime_type: str | None = None, + **kwargs: Any, + ): + """Create a Audio Content object, either from a data_uri or data. + + Args: + uri (Url | str | None): The reference uri of the content. + data_uri (DataUrl | None): The data uri of the content. + data (str | bytes | ndarray | None): The data of the content. + data_format (str | None): The format of the data (e.g. base64). + mime_type (str | None): The mime type of the image, only used with data. + kwargs (Any): Any additional arguments: + inner_content (Any): The inner content of the response, + this should hold all the information from the response so even + when not creating a subclass a developer can leverage the full thing. + ai_model_id (str | None): The id of the AI model that generated this response. + metadata (dict[str, Any]): Any metadata that should be attached to the response. + """ + super().__init__(uri=uri, data_uri=data_uri, data=data, data_format=data_format, mime_type=mime_type, **kwargs) + @classmethod def from_audio_file(cls: type[_T], path: str) -> _T: """Create an instance from an audio file.""" From 3dacf8c22a8585fb4bcb15e41bb6b0468bff2adc Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Mon, 20 Jan 2025 16:49:51 +0100 Subject: [PATCH 17/25] fix import --- .../connectors/ai/open_ai/services/open_ai_realtime.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index 9ba373cce6ff..076c46396ed7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,8 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. -from ast import TypeVar from collections.abc import Mapping -from typing import Any, ClassVar, Literal +from typing import Any, ClassVar, Literal, TypeVar from openai import AsyncOpenAI from pydantic import ValidationError From 63954197e04c4d93640b6591c17eef9d85d162f7 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Tue, 21 Jan 2025 15:29:21 +0100 Subject: [PATCH 18/25] small optimization in code --- .../audio/04-chat_with_realtime_api.py | 43 ++--- .../ai/open_ai/services/open_ai_realtime.py | 12 +- .../realtime/open_ai_realtime_base.py | 5 +- .../realtime/open_ai_realtime_webrtc.py | 11 +- .../realtime/open_ai_realtime_websocket.py | 9 +- .../connectors/ai/utils/__init__.py | 0 .../ai/{ => utils}/realtime_helpers.py | 148 +++++++++++------- .../contents/binary_content.py | 3 + 8 files changed, 133 insertions(+), 98 deletions(-) create mode 100644 python/semantic_kernel/connectors/ai/utils/__init__.py rename python/semantic_kernel/connectors/ai/{ => utils}/realtime_helpers.py (53%) diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index af4024e12849..6259d06f7061 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. import asyncio import logging -import signal +from datetime import datetime from random import randint import sounddevice as sd @@ -15,7 +15,7 @@ ) from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_websocket import ListenEvents from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.connectors.ai.realtime_helpers import SKSimplePlayer +from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioPlayer from semantic_kernel.contents import ChatHistory from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent from semantic_kernel.functions import kernel_function @@ -61,7 +61,7 @@ class ReceivingStreamHandler: It can also be used to act on other events from the service. """ - def __init__(self, realtime_client: RealtimeClientBase, audio_player: SKSimplePlayer | None = None): + def __init__(self, realtime_client: RealtimeClientBase, audio_player: SKAudioPlayer | None = None): self.audio_player = audio_player self.realtime_client = realtime_client @@ -92,12 +92,6 @@ async def listen( print("\nThanks for talking to Mosscap!") -# this function is used to stop the processes when ctrl + c is pressed -def signal_handler(): - for task in asyncio.all_tasks(): - task.cancel() - - weather_conditions = ["sunny", "hot", "cloudy", "raining", "freezing", "snowing"] @@ -109,20 +103,26 @@ def get_weather(location: str) -> str: return f"The weather in {location} is {weather}." -async def main() -> None: - # setup the asyncio loop with the signal event handler - loop = asyncio.get_event_loop() - loop.add_signal_handler(signal.SIGINT, signal_handler) +@kernel_function +def get_date_time() -> str: + """Get the current date and time.""" + return f"The current date and time is {datetime.now().isoformat()}." + +async def main() -> None: # create the Kernel and add a simple function for function calling. kernel = Kernel() kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) + kernel.add_function(plugin_name="time", function_name="get_date_time", function=get_date_time) # create the realtime client and optionally add the audio output function, this is optional - audio_player = SKSimplePlayer() - realtime_client = OpenAIRealtime(protocol="webrtc", audio_output=audio_player.realtime_client_callback) + audio_player = SKAudioPlayer() + # you can define the protocol to use, either "websocket" or "webrtc" + # they will behave the same way, even though the underlying protocol is quite different + realtime_client = OpenAIRealtime(protocol="webrtc", audio_output_callback=audio_player.client_callback) - # create stream receiver, this can play the audio, if the audio_player is passed + # create stream receiver (defined above), this can play the audio, + # if the audio_player is passed (commented out here) # and allows you to print the transcript of the conversation # and review or act on other events from the service stream_handler = ReceivingStreamHandler(realtime_client) # SimplePlayer(device_id=None) @@ -148,7 +148,7 @@ async def main() -> None: settings = OpenAIRealtimeExecutionSettings( instructions=instructions, - voice="sage", + voice="alloy", turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), function_choice_behavior=FunctionChoiceBehavior.Auto(), ) @@ -157,11 +157,12 @@ async def main() -> None: await realtime_client.update_session( settings=settings, chat_history=chat_history, kernel=kernel, create_response=True ) - # you can also send other events to the service, like this - # await realtime_client.send_buffer.put(( + # you can also send other events to the service, like this (the first has content, the second does not) + # await realtime_client.send( # SendEvents.CONVERSATION_ITEM_CREATE, - # {"item": ChatMessageContent(role="user", content="Hi there, who are you?")}, - # )) + # item=ChatMessageContent(role="user", content="Hi there, who are you?")}, + # ) + # await realtime_client.send(SendEvents.RESPONSE_CREATE) async with asyncio.TaskGroup() as tg: tg.create_task(realtime_client.start_streaming()) tg.create_task(stream_handler.listen()) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index 076c46396ed7..1a7c5acc330d 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,8 +1,9 @@ # Copyright (c) Microsoft. All rights reserved. -from collections.abc import Mapping +from collections.abc import Callable, Coroutine, Mapping from typing import Any, ClassVar, Literal, TypeVar +from numpy import ndarray from openai import AsyncOpenAI from pydantic import ValidationError @@ -31,6 +32,7 @@ def __new__(cls: type["_T"], *args: Any, **kwargs: Any) -> "_T": def __init__( self, protocol: Literal["websocket", "webrtc"] = "websocket", + audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None, ai_model_id: str | None = None, api_key: str | None = None, org_id: str | None = None, @@ -45,6 +47,13 @@ def __init__( Args: protocol: The protocol to use, can be either "websocket" or "webrtc". + audio_output_callback: The audio output callback, optional. + This should be a coroutine, that takes a ndarray with audio as input. + The goal of this function is to allow you to play the audio with the + least amount of latency possible. + It is called first in both websockets and webrtc. + Even when passed, the audio content will still be + added to the receiving queue. ai_model_id (str | None): OpenAI model name, see https://platform.openai.com/docs/models service_id (str | None): Service ID tied to the execution settings. @@ -74,6 +83,7 @@ def __init__( raise ServiceInitializationError("The OpenAI text model ID is required.") super().__init__( protocol=protocol, + audio_output_callback=audio_output_callback, ai_model_id=openai_settings.realtime_model_id, service_id=service_id, api_key=openai_settings.api_key.get_secret_value() if openai_settings.api_key else None, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py index 9f72ee1fd5d1..2865138cf0bb 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py @@ -10,6 +10,7 @@ else: from typing_extensions import override # pragma: no cover +from numpy import ndarray from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent from openai.types.beta.realtime.response_function_call_arguments_done_event import ( ResponseFunctionCallArgumentsDoneEvent, @@ -49,7 +50,7 @@ class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): protocol: ClassVar[Literal["websocket", "webrtc"]] = "websocket" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True - audio_output: Callable[[Any], Coroutine[Any, Any, None] | None] | None = None + audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None kernel: Kernel | None = None _current_settings: PromptExecutionSettings | None = PrivateAttr(None) @@ -60,8 +61,6 @@ async def _handle_event(self, event: RealtimeServerEvent) -> None: Audio delta has to be handled by the implementation of the protocol as some protocols have different ways of handling audio. - - """ match event.type: case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py index 583e34bfd997..1cfd68db0aaa 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py @@ -5,8 +5,6 @@ import json import logging import sys -from collections.abc import Callable, Coroutine -from inspect import isawaitable from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase @@ -55,7 +53,6 @@ class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase): protocol: ClassVar[Literal["webrtc"]] = "webrtc" peer_connection: RTCPeerConnection | None = None data_channel: RTCDataChannel | None = None - audio_output: Callable[[AudioFrame], Coroutine[Any, Any, None] | None] | None = None # region public methods @@ -168,7 +165,7 @@ async def create_session( ) -> None: """Create a session in the service.""" if not audio_track: - from semantic_kernel.connectors.ai.realtime_helpers import SKAudioTrack + from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioTrack audio_track = SKAudioTrack() @@ -245,10 +242,8 @@ async def _on_track(self, track: "MediaStreamTrack") -> None: break try: - if self.audio_output: - out = self.audio_output(frame) - if isawaitable(out): - await out + if self.audio_output_callback: + await self.audio_output_callback(frame.to_ndarray()) except Exception as e: logger.error(f"Error playing remote audio frame: {e!s}") diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py index 95ff1ab3a6b8..85048a4bfaef 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py @@ -5,9 +5,10 @@ import json import logging import sys -from inspect import isawaitable from typing import TYPE_CHECKING, Any, ClassVar, Literal +import numpy as np + from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase if sys.version_info >= (3, 12): @@ -62,10 +63,8 @@ async def start_listening( async for event in self.connection: if event.type == ListenEvents.RESPONSE_AUDIO_DELTA.value: - if self.audio_output: - out = self.audio_output(event) - if isawaitable(out): - await out + if self.audio_output_callback: + await self.audio_output_callback(np.frombuffer(base64.b64decode(event.delta), dtype=np.int16)) try: await self.receive_buffer.put(( event.type, diff --git a/python/semantic_kernel/connectors/ai/utils/__init__.py b/python/semantic_kernel/connectors/ai/utils/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/python/semantic_kernel/connectors/ai/realtime_helpers.py b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py similarity index 53% rename from python/semantic_kernel/connectors/ai/realtime_helpers.py rename to python/semantic_kernel/connectors/ai/utils/realtime_helpers.py index b89988f90ab3..dd6d0e5fe16f 100644 --- a/python/semantic_kernel/connectors/ai/realtime_helpers.py +++ b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py @@ -5,6 +5,7 @@ from typing import Any, Final import numpy as np +import numpy.typing as npt from aiortc.mediastreams import MediaStreamError, MediaStreamTrack from av.audio.frame import AudioFrame from av.frame import Frame @@ -20,25 +21,25 @@ TRACK_CHANNELS: Final[int] = 1 PLAYER_CHANNELS: Final[int] = 2 FRAME_DURATION: Final[int] = 20 -DTYPE: Final[np.dtype] = np.int16 +DTYPE: Final[npt.DTypeLike] = np.int16 class SKAudioTrack(KernelBaseModel, MediaStreamTrack): - """A simple class using sounddevice to record audio from the default input device. + """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. - And implementing the MediaStreamTrack interface for use with aiortc. + Make sure the device_id is set to the correct device for your system. """ kind: str = "audio" sample_rate: int = SAMPLE_RATE channels: int = TRACK_CHANNELS frame_duration: int = FRAME_DURATION - dtype: np.dtype = DTYPE + dtype: npt.DTypeLike = DTYPE device: str | int | None = None queue: asyncio.Queue[Frame] = Field(default_factory=asyncio.Queue) is_recording: bool = False - stream: InputStream | None = None frame_size: int = 0 + _stream: InputStream | None = None _recording_task: asyncio.Task | None = None _loop: asyncio.AbstractEventLoop | None = None _pts: int = 0 # Add this to track the pts @@ -62,11 +63,36 @@ async def recv(self) -> Frame: self._recording_task = asyncio.create_task(self.start_recording()) try: - return await self.queue.get() + frame = await self.queue.get() + self.queue.task_done() + return frame except Exception as e: logger.error(f"Error receiving audio frame: {e!s}") raise MediaStreamError("Failed to receive audio frame") + def _sounddevice_callback(self, indata: np.ndarray, frames: int, time: Any, status: Any) -> None: + if status: + logger.warning(f"Audio input status: {status}") + if self._loop and self._loop.is_running(): + asyncio.run_coroutine_threadsafe(self.queue.put(self._create_frame(indata)), self._loop) + + def _create_frame(self, indata: np.ndarray) -> Frame: + audio_data = indata.copy() + if audio_data.dtype != self.dtype: + audio_data = ( + (audio_data * 32767).astype(self.dtype) if self.dtype == np.int16 else audio_data.astype(self.dtype) + ) + frame = AudioFrame( + format="s16", + layout="mono", + samples=len(audio_data), + ) + frame.rate = self.sample_rate + frame.pts = self._pts + frame.planes[0].update(audio_data.tobytes()) + self._pts += len(audio_data) + return frame + async def start_recording(self): """Start recording audio from the input device.""" if self.is_recording: @@ -77,39 +103,15 @@ async def start_recording(self): self._pts = 0 # Reset pts when starting recording try: - - def callback(indata: np.ndarray, frames: int, time: Any, status: Any) -> None: - if status: - logger.warning(f"Audio input status: {status}") - - audio_data = indata.copy() - if audio_data.dtype != self.dtype: - if self.dtype == np.int16: - audio_data = (audio_data * 32767).astype(self.dtype) - else: - audio_data = audio_data.astype(self.dtype) - - frame = AudioFrame( - format="s16", - layout="mono", - samples=len(audio_data), - ) - frame.rate = self.sample_rate - frame.pts = self._pts - frame.planes[0].update(audio_data.tobytes()) - self._pts += len(audio_data) - if self._loop and self._loop.is_running(): - asyncio.run_coroutine_threadsafe(self.queue.put(frame), self._loop) - - self.stream = InputStream( + self._stream = InputStream( device=self.device, channels=self.channels, samplerate=self.sample_rate, dtype=self.dtype, blocksize=self.frame_size, - callback=callback, + callback=self._sounddevice_callback, ) - self.stream.start() + self._stream.start() while self.is_recording: await asyncio.sleep(0.1) @@ -121,7 +123,7 @@ def callback(indata: np.ndarray, frames: int, time: Any, status: Any) -> None: self.is_recording = False -class SKSimplePlayer(KernelBaseModel): +class SKAudioPlayer(KernelBaseModel): """Simple class that plays audio using sounddevice. Make sure the device_id is set to the correct device for your system. @@ -132,22 +134,12 @@ class SKSimplePlayer(KernelBaseModel): device_id: int | None = None sample_rate: int = SAMPLE_RATE + dtype: npt.DTypeLike = DTYPE channels: int = PLAYER_CHANNELS frame_duration_ms: int = FRAME_DURATION - queue: asyncio.Queue[np.ndarray] = Field(default_factory=asyncio.Queue) + _queue: asyncio.Queue[np.ndarray] | None = None _stream: OutputStream | None = PrivateAttr(None) - def model_post_init(self, __context: Any) -> None: - """Initialize the audio stream.""" - self._stream = OutputStream( - callback=self.callback, - samplerate=self.sample_rate, - channels=self.channels, - dtype=np.int16, - blocksize=int(self.sample_rate * self.frame_duration_ms / 1000), - device=self.device_id, - ) - async def __aenter__(self): """Start the audio stream when entering a context.""" self.start() @@ -159,32 +151,68 @@ async def __aexit__(self, exc_type, exc, tb): def start(self): """Start the audio stream.""" - if self._stream: + self._queue = asyncio.Queue() + self._stream = OutputStream( + callback=self._sounddevice_callback, + samplerate=self.sample_rate, + channels=self.channels, + dtype=self.dtype, + blocksize=int(self.sample_rate * self.frame_duration_ms / 1000), + device=self.device_id, + ) + if self._stream and self._queue: self._stream.start() def stop(self): """Stop the audio stream.""" if self._stream: self._stream.stop() + self._stream = None + self._queue = None - def callback(self, outdata, frames, time, status): + def _sounddevice_callback(self, outdata, frames, time, status): """This callback is called by sounddevice when it needs more audio data to play.""" if status: logger.info(f"Audio output status: {status}") - if self.queue.empty(): - return - data: np.ndarray = self.queue.get_nowait() - outdata[:] = data.reshape(outdata.shape) - - async def realtime_client_callback(self, frame: AudioFrame): - """This function is used by the RealtimeClientBase to play audio.""" - await self.queue.put(frame.to_ndarray()) + if self._queue: + if self._queue.empty(): + return + data: np.ndarray = self._queue.get_nowait() + outdata[:] = data.reshape(outdata.shape) + self._queue.task_done() + + async def client_callback(self, content: np.ndarray): + """This function can be passed to the audio_output_callback field of the RealtimeClientBase.""" + if self._queue: + await self._queue.put(content) + else: + logger.error( + "Audio queue not initialized, make sure to call start before " + "using the player, or use the context manager." + ) - async def add_audio(self, audio_content: AudioContent): + async def add_audio(self, audio_content: AudioContent) -> None: """This function is used to add audio to the queue for playing. - It uses a shortcut for this sample, because we know a AudioFrame is in the inner_content field. + It first checks if there is a AudioFrame in the inner_content of the AudioContent. + If not, it checks if the data is a numpy array, bytes, or a string and converts it to a numpy array. """ + if not self._queue: + logger.error( + "Audio queue not initialized, make sure to call start before " + "using the player, or use the context manager." + ) + return if audio_content.inner_content and isinstance(audio_content.inner_content, AudioFrame): - await self.queue.put(audio_content.inner_content.to_ndarray()) - # TODO (eavanvalkenburg): check ndarray + await self._queue.put(audio_content.inner_content.to_ndarray()) + return + if isinstance(audio_content.data, np.ndarray): + await self._queue.put(audio_content.data) + return + if isinstance(audio_content.data, bytes): + await self._queue.put(np.frombuffer(audio_content.data, dtype=self.dtype)) + return + if isinstance(audio_content.data, str): + await self._queue.put(np.frombuffer(audio_content.data.encode(), dtype=self.dtype)) + return + logger.error(f"Unknown audio content: {audio_content}") diff --git a/python/semantic_kernel/contents/binary_content.py b/python/semantic_kernel/contents/binary_content.py index 1eaa5e30217b..007186a9363e 100644 --- a/python/semantic_kernel/contents/binary_content.py +++ b/python/semantic_kernel/contents/binary_content.py @@ -176,6 +176,9 @@ def from_element(cls: type[_T], element: Element) -> _T: def write_to_file(self, path: str | FilePath) -> None: """Write the data to a file.""" + if isinstance(self.data, ndarray): + self.data.tofile(path) # codespell:ignore tofile + return with open(path, "wb") as file: file.write(self.data) From ccd84b44d031e6b7faba94cb94af69b4e035e26d Mon Sep 17 00:00:00 2001 From: Eduard van Valkenburg Date: Wed, 22 Jan 2025 13:09:30 +0100 Subject: [PATCH 19/25] updates to the ADR --- docs/decisions/00XX-realtime-api-clients.md | 49 ++++++++++----------- 1 file changed, 24 insertions(+), 25 deletions(-) diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index 6fcf0972aea2..a51744d9d400 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -12,25 +12,26 @@ informed: ## Context and Problem Statement -Multiple model providers are starting to enable realtime voice-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. The key addition that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. There are also options for Google to use video and images as input, so really it is multimodal, but for now we are focusing on the voice-to-voice part, while keeping in mind that video is coming. +Multiple model providers are starting to enable realtime voice-to-voice or even multi-model-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. + The key feature that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. There are also options for Google to use video and images as input, but for now we are focusing on the voice-to-voice part, while keeping in mind that video is coming. -The way these API's work at this time is through either Websockets or WebRTC. +The protocols that these API's use at this time are Websockets and WebRTC. In both cases there are events being sent to and from the service, some events contain content, text, audio, or video (so far only sending, not receiving), while some events are "control" events, like content created, function call requested, etc. Sending events include, sending content, either voice, text or function call output, or events, like committing the input audio and requesting a response. ### Websocket -Websocket has been around for a while and is a well known technology, it is a full-duplex communication protocol over a single, long-lived connection. It is used for sending and receiving messages between client and server in real-time. Each event can contain a message, which might contain a content item, or a control event. +Websocket has been around for a while and is a well known technology, it is a full-duplex communication protocol over a single, long-lived connection. It is used for sending and receiving messages between client and server in real-time. Each event can contain a message, which might contain a content item, or a control event. Audio is sent as a base64 encoded string. ### WebRTC -WebRTC is a Mozilla project that provides web browsers and mobile applications with real-time communication via simple application programming interfaces (APIs). It allows audio and video communication to work inside web pages by allowing direct peer-to-peer communication, eliminating the need to install plugins or download native apps. It is used for sending and receiving audio and video streams, and can be used for sending messages as well. The big difference compared to websockets is that it does explicitly create a channel for audio and video, and a separate channel for "data", which are events but also things like Function calls. +WebRTC is a Mozilla project that provides web browsers and mobile applications with real-time communication via simple application programming interfaces (APIs). It allows audio and video communication to work inside web pages and other applications by allowing direct peer-to-peer communication, eliminating the need to install plugins or download native apps. It is used for sending and receiving audio and video streams, and can be used for sending (data-)messages as well. The big difference compared to websockets is that it explicitly create a channel for audio and video, and a separate channel for "data", which are events but in this space also things like Function calls. Both the OpenAI and Google realtime api's are in preview/beta, this means there might be breaking changes in the way they work coming in the future, therefore the clients built to support these API's are going to be experimental until the API's stabilize. One feature that we need to consider if and how to deal with is whether or not a service uses Voice Activated Detection, OpenAI supports turning that off and allows parameters for how it behaves, while Google has it on by default and it cannot be configured. -### Event types (websocket and partially webrtc) +### Event types (Websocket and partially WebRTC) -Client side events: +#### Client side events: | **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | | ------------------------- | --------------------------------- | ---------------------------- | ---------------------------------- | | Control | Configure session | `session.update` | `BidiGenerateContentSetup` | @@ -44,7 +45,7 @@ Client side events: | Control | Ask for response | `response.create` | `-` | | Control | Cancel response | `response.cancel` | `-` | -Server side events: +#### Server side events: | **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | | ------------------------- | -------------------------------------- | ------------------------------------------------------- | ----------------------------------------- | | Control | Error | `error` | `-` | @@ -78,13 +79,10 @@ Server side events: | Control | Rate limits updated | `rate_limits.updated` | `-` | -## Decision Drivers -- Simple programming model that is likely able to handle future realtime api's and evolution of the existing ones. +## Overall Decision Drivers +- Simple programming model that is likely able to handle future realtime api's and the evolution of the existing ones. - Whenever possible we transform incoming content into Semantic Kernel content, but surface everything, so it's extensible -- Protocol agnostic, should be able to use different types of protocols under the covers, like websocket and WebRTC, without changing the client code (unless the protocol requires it). - -## Decision driver questions -- For WebRTC, a audio device can be passed, should this be a requirement for the client also for websockets? +- Protocol agnostic, should be able to use different types of protocols under the covers, like websocket and WebRTC, without changing the client code (unless the protocol requires it), there will be slight differences in behavior depending on the protocol. There are multiple areas where we need to make decisions, these are: - Content and Events @@ -94,7 +92,7 @@ There are multiple areas where we need to make decisions, these are: # Content and Events ## Considered Options - Content and Events -Both the sending and receiving side of these integrations need to decide how to deal with the api's. +Both the sending and receiving side of these integrations need to decide how to deal with the events. 1. Treat content events separate from control events 1. Treat everything as content items @@ -163,6 +161,16 @@ This would mean that the there are two queues, one for sending and one for recei - Con: - potentially causes audio delays because of the queueing mechanism +### 2b. Same as option 2, but with priority handling of audio content +This would mean that the audio content is handled, and passed to the developer code, and then all other events are processed. + +- Pro: + - mitigates audio delays + - easy to understand, as queues are a well known concept + - developers can just skip events they are not interested in +- Con: + - Two separate mechanisms used for audio content and events + ## Decision Outcome - Programming model Chosen option: ... @@ -172,7 +180,7 @@ Chosen option: ... ## Considered Options - Audio speaker/microphone handling 1. Create abstraction in SK for audio handlers, that can be passed into the realtime client to record and play audio -2. Send and receive AudioContent (wrapped in StreamingChatMessageContent) to the client, and let the client handle the audio recording and playing +2. Send and receive AudioContent (potentially wrapped in StreamingChatMessageContent) to the client, and let the client handle the audio recording and playing ### 1. Create abstraction in SK for audio handlers, that can be passed into the realtime client to record and play audio This would mean that the client would have a mechanism to register audio handlers, and the integration would call these handlers when audio is received or needs to be sent. A additional abstraction for this would have to be created in Semantic Kernel (or potentially taken from a standard). @@ -191,17 +199,8 @@ This would mean that the client would receive AudioContent items, and would have - no extra code in SK that needs to be maintained - Con: - extra burden on the developer to deal with the audio + - harder to get started with ## Decision Outcome - Audio speaker/microphone handling Chosen option: ... - - - -## More Information - -{You might want to provide additional evidence/confidence for the decision outcome here and/or -document the team agreement on the decision and/or -define when this decision when and how the decision should be realized and if/when it should be re-visited and/or -how the decision is validated. -Links to other decisions and resources might appear here as well.} From 3d4ad22ea3a63c9f01dac700a4fb9788fe814505 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 23 Jan 2025 11:06:40 +0100 Subject: [PATCH 20/25] import improvements --- .../audio/04-chat_with_realtime_api.py | 44 ++++++++++--------- .../connectors/ai/open_ai/__init__.py | 3 ++ .../connectors/ai/utils/__init__.py | 5 +++ 3 files changed, 31 insertions(+), 21 deletions(-) diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/04-chat_with_realtime_api.py index 6259d06f7061..a1884349b3e7 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api.py @@ -1,23 +1,21 @@ # Copyright (c) Microsoft. All rights reserved. + import asyncio import logging from datetime import datetime from random import randint -import sounddevice as sd - from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai import ( + ListenEvents, OpenAIRealtime, OpenAIRealtimeExecutionSettings, TurnDetection, ) -from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_websocket import ListenEvents from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioPlayer -from semantic_kernel.contents import ChatHistory -from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.connectors.ai.utils import SKAudioPlayer +from semantic_kernel.contents import ChatHistory, StreamingChatMessageContent from semantic_kernel.functions import kernel_function logging.basicConfig(level=logging.WARNING) @@ -47,7 +45,9 @@ def check_audio_devices(): - logger.info(sd.query_devices()) + import sounddevice as sd + + logger.debug(sd.query_devices()) check_audio_devices() @@ -87,25 +87,26 @@ async def listen( case ListenEvents.RESPONSE_CREATED: if print_transcript: print("") + # case ....: + # # add other event handling here await asyncio.sleep(0.01) except asyncio.CancelledError: print("\nThanks for talking to Mosscap!") -weather_conditions = ["sunny", "hot", "cloudy", "raining", "freezing", "snowing"] - - @kernel_function def get_weather(location: str) -> str: """Get the weather for a location.""" - weather = weather_conditions[randint(0, len(weather_conditions))] # nosec - logger.warning(f"Getting weather for {location}: {weather}") + weather_conditions = ("sunny", "hot", "cloudy", "raining", "freezing", "snowing") + weather = weather_conditions[randint(0, len(weather_conditions) - 1)] # nosec + logger.info(f"Getting weather for {location}: {weather}") return f"The weather in {location} is {weather}." @kernel_function def get_date_time() -> str: """Get the current date and time.""" + logger.info("Getting current datetime") return f"The current date and time is {datetime.now().isoformat()}." @@ -128,10 +129,6 @@ async def main() -> None: stream_handler = ReceivingStreamHandler(realtime_client) # SimplePlayer(device_id=None) # Create the settings for the session - # the key thing to decide on is to enable the server_vad turn detection - # if turn is turned off (by setting turn_detection=None), you will have to send - # the "input_audio_buffer.commit" and "response.create" event to the realtime api - # to signal the end of the user's turn and start the response. # The realtime api, does not use a system message, but takes instructions as a parameter for a session instructions = """ You are a chat bot. Your name is Mosscap and @@ -141,17 +138,22 @@ async def main() -> None: effectively, but you tend to answer with long flowery prose. """ - # and we can add a chat history to conversation after starting it - chat_history = ChatHistory() - chat_history.add_user_message("Hi there, who are you?") - chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") - + # the key thing to decide on is to enable the server_vad turn detection + # if turn is turned off (by setting turn_detection=None), you will have to send + # the "input_audio_buffer.commit" and "response.create" event to the realtime api + # to signal the end of the user's turn and start the response. + # manual VAD is not part of this sample settings = OpenAIRealtimeExecutionSettings( instructions=instructions, voice="alloy", turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), function_choice_behavior=FunctionChoiceBehavior.Auto(), ) + # and we can add a chat history to conversation after starting it + chat_history = ChatHistory() + chat_history.add_user_message("Hi there, who are you?") + chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") + # the context manager calls the create_session method on the client and start listening to the audio stream async with realtime_client, audio_player: await realtime_client.update_session( diff --git a/python/semantic_kernel/connectors/ai/open_ai/__init__.py b/python/semantic_kernel/connectors/ai/open_ai/__init__.py index 27d36ea30d34..4241ec1e49f3 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/__init__.py +++ b/python/semantic_kernel/connectors/ai/open_ai/__init__.py @@ -45,6 +45,7 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_embedding import OpenAITextEmbedding from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_audio import OpenAITextToAudio from semantic_kernel.connectors.ai.open_ai.services.open_ai_text_to_image import OpenAITextToImage +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents from semantic_kernel.connectors.ai.open_ai.settings.azure_open_ai_settings import AzureOpenAISettings from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings @@ -68,6 +69,7 @@ "DataSourceFieldsMapping", "DataSourceFieldsMapping", "ExtraBody", + "ListenEvents", "OpenAIAudioToText", "OpenAIAudioToTextExecutionSettings", "OpenAIChatCompletion", @@ -84,5 +86,6 @@ "OpenAITextToAudioExecutionSettings", "OpenAITextToImage", "OpenAITextToImageExecutionSettings", + "SendEvents", "TurnDetection", ] diff --git a/python/semantic_kernel/connectors/ai/utils/__init__.py b/python/semantic_kernel/connectors/ai/utils/__init__.py index e69de29bb2d1..2cd59106a8a0 100644 --- a/python/semantic_kernel/connectors/ai/utils/__init__.py +++ b/python/semantic_kernel/connectors/ai/utils/__init__.py @@ -0,0 +1,5 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioPlayer, SKAudioTrack + +__all__ = ["SKAudioPlayer", "SKAudioTrack"] From e68bd827bbcdf34c4eb4a15ebbfcf70c106d09c7 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Tue, 28 Jan 2025 16:02:31 +0100 Subject: [PATCH 21/25] updated code and ADR --- docs/decisions/00XX-realtime-api-clients.md | 41 ++++++++- .../audio/04-chat_with_realtime_api_simple.py | 88 ++++++++++++++++++ ...y => 05-chat_with_realtime_api_complex.py} | 73 +++++---------- .../realtime/open_ai_realtime_base.py | 92 +++++++++++-------- .../realtime/open_ai_realtime_webrtc.py | 62 +++++++------ .../realtime/open_ai_realtime_websocket.py | 45 ++++----- .../connectors/ai/realtime_client_base.py | 19 ++-- .../contents/realtime_event.py | 56 +++++++++++ 8 files changed, 323 insertions(+), 153 deletions(-) create mode 100644 python/samples/concepts/audio/04-chat_with_realtime_api_simple.py rename python/samples/concepts/audio/{04-chat_with_realtime_api.py => 05-chat_with_realtime_api_complex.py} (68%) create mode 100644 python/semantic_kernel/contents/realtime_event.py diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index a51744d9d400..94c7351ba7fe 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -128,7 +128,30 @@ This would mean that all events are retained and returned to the developer as is ## Decision Outcome - Content and Events -Chosen option: ... +Chosen option: 3 Treat Everything as Events + +This option was chosen to allow abstraction away from the raw events, while still allowing the developer to access the raw events if needed. This allows for a simple programming model, while still allowing for complex interactions. +A set of events are defined, for basic types, like 'audio', 'text', 'function_call', 'function_result', it then has two other fields, service_event which is filled with the event type from the service and a field for the actual content, with a name that makes sense: + +```python +AudioEvent( + event_type="audio", + service_event= "response.audio.delta", + audio: AudioContent(...) +) +``` + +Next to these we will have a generic event, called ServiceEvent, this is the catch-all, which has event_type: "service", the service_event field filled with the event type from the service and a field called 'event' which contains the raw event from the service. + +```python +ServiceEvent( + event_type="service", + service_event= "conversation.item.create", + event: { ... } +) +``` + +This allows you to easily filter on the event_type, and then use the service_event to filter on the specific event type, and then use the content field to get the content, or the event field to get the raw event. # Programming model @@ -137,10 +160,11 @@ The programming model for the clients needs to be simple and easy to use, while _In this section we will refer to events for both content and events, regardless of the decision made in the previous section._ -1. Async generator for receiving events, that yields contents, combined with a event handler/callback mechanism for receiving events and a function for sending events +1. Async generator for receiving events, that yields Events, combined with a event handler/callback mechanism for receiving events and a function for sending events - 1a: Single event handlers, where each event is passed to the handler - 1b: Multiple event handlers, where each event type has its own handler 2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers +3. Purely a start listening method that yields Events, and a send method that sends events ### 1. Async generator for receiving events, that yields contents, combined with a event handler/callback mechanism for receiving events and a function for sending events This would mean that the client would have a mechanism to register event handlers, and the integration would call these handlers when an event is received. For sending events, a function would be created that sends the event to the service. @@ -173,7 +197,18 @@ This would mean that the audio content is handled, and passed to the developer c ## Decision Outcome - Programming model -Chosen option: ... +Chosen option: Purely a start listening method that yields Events, and a send method that sends events + +This makes the programming model very easy, a minimal setup that should work for every service and protocol would look like this: +```python +async for event in realtime_client.start_streaming(): + match event.event_type: + case "audio": + await audio_player.add_audio(event.audio) + case "text": + print(event.text.text) +``` + # Audio speaker/microphone handling diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py b/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py new file mode 100644 index 000000000000..116f9e3f8d81 --- /dev/null +++ b/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py @@ -0,0 +1,88 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging + +from semantic_kernel.connectors.ai.open_ai import ( + OpenAIRealtime, + OpenAIRealtimeExecutionSettings, + TurnDetection, +) +from semantic_kernel.connectors.ai.utils import SKAudioPlayer + +logging.basicConfig(level=logging.WARNING) +aiortc_log = logging.getLogger("aiortc") +aiortc_log.setLevel(logging.WARNING) +aioice_log = logging.getLogger("aioice") +aioice_log.setLevel(logging.WARNING) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +# This simple sample demonstrates how to use the OpenAI Realtime API to create +# a chat bot that can listen and respond directly through audio. +# It requires installing: +# - semantic-kernel[openai_realtime] +# - pyaudio +# - sounddevice +# - pydub +# - aiortc +# e.g. pip install pyaudio sounddevice pydub + +# The characterics of your speaker and microphone are a big factor in a smooth conversation +# so you may need to try out different devices for each. +# you can also play around with the turn_detection settings to get the best results. +# It has device id's set in the AudioRecorderStream and AudioPlayerAsync classes, +# so you may need to adjust these for your system. +# you can check the available devices by uncommenting line below the function + + +def check_audio_devices(): + import sounddevice as sd + + logger.debug(sd.query_devices()) + + +check_audio_devices() + + +async def main() -> None: + # create the realtime client and optionally add the audio output function, this is optional + # you can define the protocol to use, either "websocket" or "webrtc" + # they will behave the same way, even though the underlying protocol is quite different + realtime_client = OpenAIRealtime(protocol="webrtc") + # Create the settings for the session + settings = OpenAIRealtimeExecutionSettings( + instructions=""" + You are a chat bot. Your name is Mosscap and + you have one goal: figure out what people need. + Your full name, should you need to know it, is + Splendid Speckled Mosscap. You communicate + effectively, but you tend to answer with long + flowery prose. + """, + voice="alloy", + turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), + ) + # the context manager calls the create_session method on the client and start listening to the audio stream + audio_player = SKAudioPlayer() + async with realtime_client, audio_player: + await realtime_client.update_session(settings=settings, create_response=True) + async for event in realtime_client.start_streaming(): + match event.event_type: + case "audio": + await audio_player.add_audio(event.audio) + case "text": + print(event.text.text) + case "service": + if event.service_type == "session.update": + print("Session updated") + if event.service_type == "error": + logger.error(event.event) + + +if __name__ == "__main__": + print( + "Instruction: start speaking, when you stop the API should detect you finished and start responding. " + "Press ctrl + c to stop the program." + ) + asyncio.run(main()) diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api.py b/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py similarity index 68% rename from python/samples/concepts/audio/04-chat_with_realtime_api.py rename to python/samples/concepts/audio/05-chat_with_realtime_api_complex.py index a1884349b3e7..18785b81348d 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api.py +++ b/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py @@ -13,9 +13,8 @@ OpenAIRealtimeExecutionSettings, TurnDetection, ) -from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.connectors.ai.utils import SKAudioPlayer -from semantic_kernel.contents import ChatHistory, StreamingChatMessageContent +from semantic_kernel.contents import ChatHistory from semantic_kernel.functions import kernel_function logging.basicConfig(level=logging.WARNING) @@ -53,47 +52,6 @@ def check_audio_devices(): check_audio_devices() -class ReceivingStreamHandler: - """This is a simple class that listens to the received buffer of the RealtimeClientBase. - - It can be used to play audio and print the transcript of the conversation. - - It can also be used to act on other events from the service. - """ - - def __init__(self, realtime_client: RealtimeClientBase, audio_player: SKAudioPlayer | None = None): - self.audio_player = audio_player - self.realtime_client = realtime_client - - async def listen( - self, - play_audio: bool = True, - print_transcript: bool = True, - ) -> None: - # print the start message of the transcript - if print_transcript: - print("Mosscap (transcript): ", end="") - try: - # start listening for events - while True: - event_type, event = await self.realtime_client.receive_buffer.get() - match event_type: - case ListenEvents.RESPONSE_AUDIO_DELTA: - if play_audio and self.audio_player and isinstance(event, StreamingChatMessageContent): - await self.audio_player.add_audio(event.items[0]) - case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA: - if print_transcript and isinstance(event, StreamingChatMessageContent): - print(event.content, end="") - case ListenEvents.RESPONSE_CREATED: - if print_transcript: - print("") - # case ....: - # # add other event handling here - await asyncio.sleep(0.01) - except asyncio.CancelledError: - print("\nThanks for talking to Mosscap!") - - @kernel_function def get_weather(location: str) -> str: """Get the weather for a location.""" @@ -111,6 +69,7 @@ def get_date_time() -> str: async def main() -> None: + print_transcript = True # create the Kernel and add a simple function for function calling. kernel = Kernel() kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) @@ -122,12 +81,6 @@ async def main() -> None: # they will behave the same way, even though the underlying protocol is quite different realtime_client = OpenAIRealtime(protocol="webrtc", audio_output_callback=audio_player.client_callback) - # create stream receiver (defined above), this can play the audio, - # if the audio_player is passed (commented out here) - # and allows you to print the transcript of the conversation - # and review or act on other events from the service - stream_handler = ReceivingStreamHandler(realtime_client) # SimplePlayer(device_id=None) - # Create the settings for the session # The realtime api, does not use a system message, but takes instructions as a parameter for a session instructions = """ @@ -165,9 +118,25 @@ async def main() -> None: # item=ChatMessageContent(role="user", content="Hi there, who are you?")}, # ) # await realtime_client.send(SendEvents.RESPONSE_CREATE) - async with asyncio.TaskGroup() as tg: - tg.create_task(realtime_client.start_streaming()) - tg.create_task(stream_handler.listen()) + print("Mosscap (transcript): ", end="") + async for event in realtime_client.start_streaming(): + match event.event_type: + # case "audio": + # if play_audio and audio_player: + # await audio_player.add_audio(event.audio) + case "text": + if print_transcript: + print(event.text.text, end="") + case "service": + # OpenAI Specific events + match event.service_type: + case ListenEvents.RESPONSE_CREATED: + if print_transcript: + print("") + case ListenEvents.ERROR: + logger.error(event.event) + # case ....: + # # add other event handling here if __name__ == "__main__": diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py index 2865138cf0bb..5f2b49020fb6 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py @@ -2,9 +2,19 @@ import logging import sys -from collections.abc import Callable, Coroutine +from collections.abc import AsyncGenerator, Callable, Coroutine from typing import TYPE_CHECKING, Any, ClassVar, Literal +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.realtime_event import ( + FunctionCallEvent, + FunctionResultEvent, + RealtimeEvent, + ServiceEvent, + TextEvent, +) +from semantic_kernel.contents.streaming_text_content import StreamingTextContent + if sys.version_info >= (3, 12): from typing import override # pragma: no cover else: @@ -31,8 +41,6 @@ from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.contents.chat_history import ChatHistory from semantic_kernel.contents.function_call_content import FunctionCallContent -from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent -from semantic_kernel.contents.utils.author_role import AuthorRole from semantic_kernel.kernel import Kernel from semantic_kernel.utils.experimental_decorator import experimental_class @@ -56,7 +64,7 @@ class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): _current_settings: PromptExecutionSettings | None = PrivateAttr(None) _call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict) - async def _handle_event(self, event: RealtimeServerEvent) -> None: + async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[RealtimeEvent, None]: """Handle all events but audio delta. Audio delta has to be handled by the implementation of the protocol as some @@ -64,38 +72,35 @@ async def _handle_event(self, event: RealtimeServerEvent) -> None: """ match event.type: case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value: - await self.receive_buffer.put(( - event.type, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - content=event.delta, - choice_index=event.content_index, + yield TextEvent( + event_type="text", + service_type=event.type, + text=StreamingTextContent( inner_content=event, + text=event.delta, + choice_index=0, ), - )) + ) case ListenEvents.RESPONSE_OUTPUT_ITEM_ADDED.value: if event.item.type == "function_call" and event.item.call_id and event.item.name: self._call_id_to_function_map[event.item.call_id] = event.item.name case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA.value: - await self.receive_buffer.put(( - event.type, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[ - FunctionCallContent( - id=event.item_id, - name=event.call_id, - arguments=event.delta, - index=event.output_index, - metadata={"call_id": event.call_id}, - ) - ], - choice_index=0, + yield FunctionCallEvent( + event_type="function_call", + service_type=event.type, + function_call=FunctionCallContent( + id=event.item_id, + name=self._call_id_to_function_map[event.call_id], + arguments=event.delta, + index=event.output_index, + metadata={"call_id": event.call_id}, inner_content=event, ), - )) + ) case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE.value: - await self._handle_function_call_arguments_done(event) + async for parsed_event in self._parse_function_call_arguments_done(event): + if parsed_event: + yield parsed_event case ListenEvents.ERROR.value: logger.error("Error received: %s", event.error) case ListenEvents.SESSION_CREATED.value, ListenEvents.SESSION_UPDATED.value: @@ -105,7 +110,7 @@ async def _handle_event(self, event: RealtimeServerEvent) -> None: # we put all event in the output buffer, but after the interpreted one. # so when dealing with them, make sure to check the type of the event, since they # might be of different types. - await self.receive_buffer.put((event.type, event)) + yield ServiceEvent(event_type="service", service_type=event.type, event=event) @override async def update_session( @@ -126,12 +131,16 @@ async def update_session( self._update_function_choice_settings_callback(), kernel=self.kernel, # type: ignore ) - await self.send(SendEvents.SESSION_UPDATE, settings=self._current_settings) + await self.send( + ServiceEvent(event_type="service", service_type=SendEvents.SESSION_UPDATE, event=self._current_settings) + ) if chat_history and len(chat_history) > 0: for msg in chat_history.messages: - await self.send(SendEvents.CONVERSATION_ITEM_CREATE, item=msg) + await self.send( + ServiceEvent(event_type="service", service_type=SendEvents.CONVERSATION_ITEM_CREATE, event=msg) + ) if create_response: - await self.send(SendEvents.RESPONSE_CREATE) + await self.send(ServiceEvent(event_type="service", service_type=SendEvents.RESPONSE_CREATE)) @override def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: @@ -147,20 +156,22 @@ def _update_function_choice_settings_callback( ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: return update_settings_from_function_call_configuration - async def _handle_function_call_arguments_done( + async def _parse_function_call_arguments_done( self, event: ResponseFunctionCallArgumentsDoneEvent, - ) -> None: + ) -> AsyncGenerator[RealtimeEvent | None]: """Handle response function call done.""" if not self.kernel or ( self._current_settings and self._current_settings.function_choice_behavior and not self._current_settings.function_choice_behavior.auto_invoke_kernel_functions ): + yield None return plugin_name, function_name = self._call_id_to_function_map.pop(event.call_id, "-").split("-", 1) if not plugin_name or not function_name: logger.error("Function call needs to have a plugin name and function name") + yield None return item = FunctionCallContent( id=event.item_id, @@ -170,15 +181,22 @@ async def _handle_function_call_arguments_done( index=event.output_index, metadata={"call_id": event.call_id}, ) + yield FunctionCallEvent( + event_type="function_call", + service_type=ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, + function_call=item, + ) chat_history = ChatHistory() await self.kernel.invoke_function_call(item, chat_history) - created_output = chat_history.messages[-1] + created_output: FunctionResultContent = chat_history.messages[-1].items[0] # type: ignore # This returns the output to the service - await self.send(SendEvents.CONVERSATION_ITEM_CREATE, item=created_output) + await self.send( + ServiceEvent(event_type="service", service_type=SendEvents.CONVERSATION_ITEM_CREATE, event=created_output) + ) # The model doesn't start responding to the tool call automatically, so triggering it here. - await self.send(SendEvents.RESPONSE_CREATE) + await self.send(ServiceEvent(event_type="service", service_type=SendEvents.RESPONSE_CREATE)) # This allows a user to have a full conversation in his code - await self.receive_buffer.put((ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, created_output)) + yield FunctionResultEvent(event_type="function_result", function_result=created_output) @override async def start_listening( diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py index 1cfd68db0aaa..1225c2927345 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py @@ -5,9 +5,10 @@ import json import logging import sys +from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast -from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase +from pydantic import Field if sys.version_info >= (3, 12): from typing import override # pragma: no cover @@ -28,12 +29,13 @@ from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeEvent from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.function_call_content import FunctionCallContent from semantic_kernel.contents.function_result_content import FunctionResultContent -from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent +from semantic_kernel.contents.realtime_event import AudioEvent from semantic_kernel.contents.text_content import TextContent -from semantic_kernel.contents.utils.author_role import AuthorRole from semantic_kernel.utils.experimental_decorator import experimental_class if TYPE_CHECKING: @@ -53,6 +55,7 @@ class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase): protocol: ClassVar[Literal["webrtc"]] = "webrtc" peer_connection: RTCPeerConnection | None = None data_channel: RTCDataChannel | None = None + receive_buffer: asyncio.Queue[RealtimeEvent] = Field(default_factory=asyncio.Queue) # region public methods @@ -63,9 +66,12 @@ async def start_listening( chat_history: "ChatHistory | None" = None, create_response: bool = False, **kwargs: Any, - ) -> None: + ) -> AsyncGenerator[RealtimeEvent, None]: if chat_history or settings or create_response: await self.update_session(settings=settings, chat_history=chat_history, create_response=create_response) + while True: + event = await self.receive_buffer.get() + yield event @override async def start_sending(self, **kwargs: Any) -> None: @@ -75,20 +81,18 @@ async def start_sending(self, **kwargs: Any) -> None: while self.data_channel.readyState != "open": await asyncio.sleep(0.1) while True: - event, data = await self.send_buffer.get() - if not isinstance(event, SendEvents): - event = SendEvents(event) - response: dict[str, Any] = {"type": event.value} - match event: + event = await self.send_buffer.get() + response: dict[str, Any] = {"type": event.event_type} + match event.event_type: case SendEvents.SESSION_UPDATE: - if "settings" not in data: + if "settings" not in event.data: logger.error("Event data does not contain 'settings'") - response["session"] = data["settings"].prepare_settings_dict() + response["session"] = event.data["settings"].prepare_settings_dict() case SendEvents.CONVERSATION_ITEM_CREATE: - if "item" not in data: + if "item" not in event.data: logger.error("Event data does not contain 'item'") return - content = data["item"] + content = event.data["item"] for item in content.items: match item: case TextContent(): @@ -131,24 +135,24 @@ async def start_sending(self, **kwargs: Any) -> None: ) case SendEvents.CONVERSATION_ITEM_TRUNCATE: - if "item_id" not in data: + if "item_id" not in event.data: logger.error("Event data does not contain 'item_id'") return - response["item_id"] = data["item_id"] + response["item_id"] = event.data["item_id"] response["content_index"] = 0 - response["audio_end_ms"] = data.get("audio_end_ms", 0) + response["audio_end_ms"] = event.data.get("audio_end_ms", 0) case SendEvents.CONVERSATION_ITEM_DELETE: - if "item_id" not in data: + if "item_id" not in event.data: logger.error("Event data does not contain 'item_id'") return - response["item_id"] = data["item_id"] + response["item_id"] = event.data["item_id"] case SendEvents.RESPONSE_CREATE: - if "response" in data: - response["response"] = data["response"] + if "response" in event.data: + response["response"] = event.data["response"] case SendEvents.RESPONSE_CANCEL: - if "response_id" in data: - response["response_id"] = data["response_id"] + if "response_id" in event.data: + response["response_id"] = event.data["response_id"] try: self.data_channel.send(json.dumps(response)) @@ -249,13 +253,10 @@ async def _on_track(self, track: "MediaStreamTrack") -> None: logger.error(f"Error playing remote audio frame: {e!s}") try: await self.receive_buffer.put( - ( - ListenEvents.RESPONSE_AUDIO_DELTA, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame)], # type: ignore - choice_index=0, - ), + AudioEvent( + event_type="audio", + service_type=ListenEvents.RESPONSE_AUDIO_DELTA, + audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame), # type: ignore ), ) except Exception as e: @@ -276,7 +277,8 @@ async def _on_data(self, data: str) -> None: except Exception as e: logger.error(f"Failed to parse event {data} with error: {e!s}") return - await self._handle_event(event) + async for parsed_event in self._parse_event(event): + await self.receive_buffer.put(parsed_event) async def _get_ephemeral_token(self) -> str: """Get an ephemeral token from OpenAI.""" diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py index 85048a4bfaef..4f32067ba3cc 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py @@ -5,12 +5,11 @@ import json import logging import sys +from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, ClassVar, Literal import numpy as np -from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase - if sys.version_info >= (3, 12): from typing import override # pragma: no cover else: @@ -21,6 +20,7 @@ from pydantic import Field from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.function_call_content import FunctionCallContent from semantic_kernel.contents.function_result_content import FunctionResultContent @@ -53,7 +53,7 @@ async def start_listening( chat_history: "ChatHistory | None" = None, create_response: bool = False, **kwargs: Any, - ) -> None: + ) -> AsyncGenerator[tuple[str, Any], None]: await self.connected.wait() if not self.connection: raise ValueError("Connection is not established.") @@ -66,7 +66,7 @@ async def start_listening( if self.audio_output_callback: await self.audio_output_callback(np.frombuffer(base64.b64decode(event.delta), dtype=np.int16)) try: - await self.receive_buffer.put(( + yield ( event.type, StreamingChatMessageContent( role=AuthorRole.ASSISTANT, @@ -79,11 +79,12 @@ async def start_listening( ], # type: ignore choice_index=event.content_index, ), - )) + ) except Exception as e: logger.error(f"Error processing remote audio frame: {e!s}") else: - await self._handle_event(event) + async for event in self._parse_event(event): + yield event @override async def start_sending(self, **kwargs: Any) -> None: @@ -91,26 +92,26 @@ async def start_sending(self, **kwargs: Any) -> None: if not self.connection: raise ValueError("Connection is not established.") while True: - event, data = await self.send_buffer.get() - match event: + event = await self.send_buffer.get() + match event.event_type: case SendEvents.SESSION_UPDATE: - if "settings" not in data: + if "settings" not in event.data: logger.error("Event data does not contain 'settings'") - await self.connection.session.update(session=data["settings"].prepare_settings_dict()) + await self.connection.session.update(session=event.data["settings"].prepare_settings_dict()) case SendEvents.INPUT_AUDIO_BUFFER_APPEND: - if "content" not in data: + if "content" not in event.data: logger.error("Event data does not contain 'content'") return - await self.connection.input_audio_buffer.append(audio=data["content"].data.decode("utf-8")) + await self.connection.input_audio_buffer.append(audio=event.data["content"].data.decode("utf-8")) case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: await self.connection.input_audio_buffer.commit() case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: await self.connection.input_audio_buffer.clear() case SendEvents.CONVERSATION_ITEM_CREATE: - if "item" not in data: + if "item" not in event.data: logger.error("Event data does not contain 'item'") return - content = data["item"] + content = event.data["item"] for item in content.items: match item: case TextContent(): @@ -156,25 +157,25 @@ async def start_sending(self, **kwargs: Any) -> None: ) ) case SendEvents.CONVERSATION_ITEM_TRUNCATE: - if "item_id" not in data: + if "item_id" not in event.data: logger.error("Event data does not contain 'item_id'") return await self.connection.conversation.item.truncate( - item_id=data["item_id"], content_index=0, audio_end_ms=data.get("audio_end_ms", 0) + item_id=event.data["item_id"], content_index=0, audio_end_ms=event.data.get("audio_end_ms", 0) ) case SendEvents.CONVERSATION_ITEM_DELETE: - if "item_id" not in data: + if "item_id" not in event.data: logger.error("Event data does not contain 'item_id'") return - await self.connection.conversation.item.delete(item_id=data["item_id"]) + await self.connection.conversation.item.delete(item_id=event.data["item_id"]) case SendEvents.RESPONSE_CREATE: - if "response" in data: - await self.connection.response.create(response=data["response"]) + if "response" in event.data: + await self.connection.response.create(response=event.data["response"]) else: await self.connection.response.create() case SendEvents.RESPONSE_CANCEL: - if "response_id" in data: - await self.connection.response.cancel(response_id=data["response_id"]) + if "response_id" in event.data: + await self.connection.response.cancel(response_id=event.data["response_id"]) else: await self.connection.response.cancel() diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index 5f6fa302d545..a6a332791293 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -3,7 +3,7 @@ import sys from abc import ABC, abstractmethod from asyncio import Queue -from collections.abc import Callable +from collections.abc import AsyncGenerator, Callable from typing import TYPE_CHECKING, Any, ClassVar from pydantic import Field @@ -15,6 +15,7 @@ from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType +from semantic_kernel.contents.realtime_event import RealtimeEvent from semantic_kernel.services.ai_service_client_base import AIServiceClientBase from semantic_kernel.utils.experimental_decorator import experimental_class @@ -28,24 +29,23 @@ class RealtimeClientBase(AIServiceClientBase, ABC): """Base class for a realtime client.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False - send_buffer: Queue[tuple[str, Any]] = Field(default_factory=Queue) - receive_buffer: Queue[tuple[str, Any]] = Field(default_factory=Queue) + send_buffer: Queue[RealtimeEvent] = Field(default_factory=Queue) - async def send(self, event: str, **kwargs: Any) -> None: + async def send(self, event: RealtimeEvent) -> None: """Send an event to the service. Args: event: The event to send. kwargs: Additional arguments. """ - await self.send_buffer.put((event, kwargs)) + await self.send_buffer.put(event) async def start_streaming( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, - ) -> None: + ) -> AsyncGenerator[RealtimeEvent, None]: """Start streaming, will start both listening and sending. This method, start tasks for both listening and sending. @@ -57,9 +57,10 @@ async def start_streaming( chat_history: Chat history. kwargs: Additional arguments. """ + await self.update_session(settings=settings, chat_history=chat_history, **kwargs) async with TaskGroup() as tg: - tg.create_task(self.start_listening(settings=settings, chat_history=chat_history, **kwargs)) tg.create_task(self.start_sending(**kwargs)) + yield from tg.create_task(self.start_listening()) @abstractmethod async def start_listening( @@ -67,8 +68,8 @@ async def start_listening( settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, **kwargs: Any, - ) -> None: - """Starts listening for messages from the service, adds them to the output_buffer. + ) -> AsyncGenerator[RealtimeEvent, None]: + """Starts listening for messages from the service, generates events. Args: settings: Prompt execution settings. diff --git a/python/semantic_kernel/contents/realtime_event.py b/python/semantic_kernel/contents/realtime_event.py new file mode 100644 index 000000000000..7de87f078ff6 --- /dev/null +++ b/python/semantic_kernel/contents/realtime_event.py @@ -0,0 +1,56 @@ +# Copyright (c) Microsoft. All rights reserved. + +from typing import Annotated, Any, Literal, TypeAlias, Union + +from pydantic import Field + +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.kernel_pydantic import KernelBaseModel + +RealtimeEvent: TypeAlias = Annotated[ + Union["ServiceEvent", "AudioEvent", "TextEvent", "FunctionCallEvent", "FunctionResultEvent"], + Field(discriminator="event_type"), +] + + +class ServiceEvent(KernelBaseModel): + """Base class for all service events.""" + + event_type: Literal["service"] + service_type: str + event: Any | None = None + + +class AudioEvent(KernelBaseModel): + """Audio event type.""" + + event_type: Literal["audio"] + service_type: str | None = None + audio: AudioContent + + +class TextEvent(KernelBaseModel): + """Text event type.""" + + event_type: Literal["text"] + service_type: str | None = None + text: TextContent + + +class FunctionCallEvent(KernelBaseModel): + """Function call event type.""" + + event_type: Literal["function_call"] + service_type: str | None = None + function_call: FunctionCallContent + + +class FunctionResultEvent(KernelBaseModel): + """Function result event type.""" + + event_type: Literal["function_result"] + service_type: str | None = None + function_result: FunctionResultContent From d4510239e0b9e2d45000f2401a5ed533d586fcdb Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Wed, 29 Jan 2025 17:00:21 +0100 Subject: [PATCH 22/25] wip on redoing the api --- docs/decisions/00XX-realtime-api-clients.md | 111 ++++++++- .../audio/04-chat_with_realtime_api_simple.py | 23 +- .../05-chat_with_realtime_api_complex.py | 33 +-- python/samples/concepts/audio/utils.py | 11 + .../ai/open_ai/services/open_ai_realtime.py | 11 +- .../realtime/open_ai_realtime_base.py | 233 +++++++++++++++--- .../realtime/open_ai_realtime_webrtc.py | 133 ++-------- .../realtime/open_ai_realtime_websocket.py | 118 +-------- .../ai/open_ai/services/realtime/utils.py | 80 ++++++ .../connectors/ai/realtime_client_base.py | 50 +--- .../connectors/ai/utils/realtime_helpers.py | 91 ++++--- .../contents/events/__init__.py | 19 ++ .../contents/{ => events}/realtime_event.py | 0 13 files changed, 551 insertions(+), 362 deletions(-) create mode 100644 python/samples/concepts/audio/utils.py create mode 100644 python/semantic_kernel/contents/events/__init__.py rename python/semantic_kernel/contents/{ => events}/realtime_event.py (100%) diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md index 94c7351ba7fe..bde864d79b52 100644 --- a/docs/decisions/00XX-realtime-api-clients.md +++ b/docs/decisions/00XX-realtime-api-clients.md @@ -130,8 +130,8 @@ This would mean that all events are retained and returned to the developer as is Chosen option: 3 Treat Everything as Events -This option was chosen to allow abstraction away from the raw events, while still allowing the developer to access the raw events if needed. This allows for a simple programming model, while still allowing for complex interactions. -A set of events are defined, for basic types, like 'audio', 'text', 'function_call', 'function_result', it then has two other fields, service_event which is filled with the event type from the service and a field for the actual content, with a name that makes sense: +This option was chosen to allow abstraction away from the raw events, while still allowing the developer to access the raw events if needed. +A set of events are defined, for basic types, like 'audio', 'text', 'function_call', 'function_result', it then has two other fields, service_event which is filled with the event type from the service and a field for the actual content, with a name that corresponds to the event type: ```python AudioEvent( @@ -153,6 +153,15 @@ ServiceEvent( This allows you to easily filter on the event_type, and then use the service_event to filter on the specific event type, and then use the content field to get the content, or the event field to get the raw event. +Collectively these are known as *RealtimeEvents*, and are returned as an async generator from the client, so you can easily loop over them. And they are passed to the send method. + +Initially RealtimeEvents are: +- AudioEvent +- TextEvent +- FunctionCallEvent +- FunctionResultEvent +- ServiceEvent + # Programming model ## Considered Options - Programming model @@ -176,7 +185,7 @@ This would mean that the client would have a mechanism to register event handler - developer judgement needs to be made (or exposed with parameters) on what is returned through the async generator and what is passed to the event handlers ### 2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers -This would mean that the there are two queues, one for sending and one for receiving, and the developer can listen to the receiving queue and send to the sending queue. Internal things like parsing events to content types and auto-function calling are processed first, and the result is put in the queue, the content type should use inner_content to capture the full event and these might add a message to the send queue as well. +This would mean that there are two queues, one for sending and one for receiving, and the developer can listen to the receiving queue and send to the sending queue. Internal things like parsing events to content types and auto-function calling are processed first, and the result is put in the queue, the content type should use inner_content to capture the full event and these might add a message to the send queue as well. - Pro: - simple to use, just start sending and start receiving @@ -239,3 +248,99 @@ This would mean that the client would receive AudioContent items, and would have ## Decision Outcome - Audio speaker/microphone handling Chosen option: ... + +# Interface design + +## Considered Options - Interface design + +1. Use a single class for everything +2. Split the service class from a session class. + +The following methods will need to be supported: +- create session +- update session +- close session +- listen for/receive events +- send events + +### 1. Use a single class for everything + +Each implementation would have to implements all of the above methods. This means that non-protocol specific elements are in the same class as the protocol specific elements and will lead to code duplication between them. + +### 2. Split the service class from a session class. + +Two interfaces are created: +- Service: create session, update session, delete session, list sessions +- Session: listen for/receive events, send events, update session, close session + +Currently neither the google or the openai api's support restarting sessions, so the advantage of splitting is mostly a implementation question but will not add any benefits to the user. + +This means that the split would be far simpler: +- Service: create session +- Session: listen for/receive events, send events, update session, close session + +## Naming + +The send and listen/receive methods need to be clear in the way their are named and this can become confusing when dealing with these api's. The following options are considered: + +Options for sending events to the service from your code: +- google uses .send in their client. +- OpenAI uses .send in their client as well +- send or send_message is used in other clients, like Azure Communication Services + +Options for listening for events from the service in your code: +- google uses .receive in their client. +- openai uses .recv in their client. +- others use receive or receive_messages in their clients. + +### Decision Outcome - Interface design + +Chosen option: Use a single class for everything +Chosen for send and receive as verbs. + +This means that the interface will look like this: +```python + +class RealtimeClient: + async def create_session(self, settings: PromptExecutionSettings, chat_history: ChatHistory, **kwargs) -> None: + ... + + async def update_session(self, settings: PromptExecutionSettings, chat_history: ChatHistory, **kwargs) -> None: + ... + + async def close_session(self, **kwargs) -> None: + ... + + async def receive(self, **kwargs) -> AsyncGenerator[RealtimeEvent, None]: + ... + + async def send(self, event: RealtimeEvent) -> None: + ... +``` + +In most cases, create_session should call update_session with the same parameters, since update session can also be done separately later on with the same inputs. + +For Python a default __aenter__ and __aexit__ method should be added to the class, so it can be used in a with statement, which calls create_session and close_session respectively. + +It is advisable, but not required, to implement the send method through a buffer/queue so that events be can 'sent' before the sessions has been established without losing them or raising exceptions, this might take a very seconds and in that time a single send call would block the application. + +For receiving a internal implementation might also rely on a buffer/queue, but this is up to the developer and what makes sense for that service. For instance webrtc relies on defining the callback at create session time, so the create_session method adds a function that adds events to the queue and the receive method starts reading from and yielding from that queue. + +The send method should handle all events types, but it might have to handle the same thing in two ways, for instance: +```python +audio = AudioContent(...) + +await client.send(AudioEvent(event_type='audio', audio=audio)) +``` + +is equivalent to (at least in the case of OpenAI): +```python +audio = AudioContent(...) + +await client.send(ServiceEvent(event_type='service', service_event='input_audio_buffer.append', event=audio)) +``` + +The first version allows one to have the exact same code for all services, while the second version is also correct and should be handled correctly as well, this once again allows for flexibility and simplicity, when audio needs to be sent to with a different event type, that is still possible in the second way, while the first uses the "default" event type for that particular service. + + + diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py b/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py index 116f9e3f8d81..5dda1dc6d308 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py @@ -3,7 +3,9 @@ import asyncio import logging +from samples.concepts.audio.utils import check_audio_devices from semantic_kernel.connectors.ai.open_ai import ( + ListenEvents, OpenAIRealtime, OpenAIRealtimeExecutionSettings, TurnDetection, @@ -34,14 +36,6 @@ # It has device id's set in the AudioRecorderStream and AudioPlayerAsync classes, # so you may need to adjust these for your system. # you can check the available devices by uncommenting line below the function - - -def check_audio_devices(): - import sounddevice as sd - - logger.debug(sd.query_devices()) - - check_audio_devices() @@ -65,18 +59,23 @@ async def main() -> None: ) # the context manager calls the create_session method on the client and start listening to the audio stream audio_player = SKAudioPlayer() + print("Mosscap (transcript): ", end="") async with realtime_client, audio_player: await realtime_client.update_session(settings=settings, create_response=True) - async for event in realtime_client.start_streaming(): + + async for event in realtime_client.receive(): match event.event_type: case "audio": await audio_player.add_audio(event.audio) case "text": - print(event.text.text) + print(event.text.text, end="") case "service": - if event.service_type == "session.update": + # OpenAI Specific events + if event.service_type == ListenEvents.SESSION_UPDATED: print("Session updated") - if event.service_type == "error": + if event.service_type == ListenEvents.RESPONSE_CREATED: + print("") + if event.service_type == ListenEvents.ERROR: logger.error(event.event) diff --git a/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py b/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py index 18785b81348d..77406af4f355 100644 --- a/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py +++ b/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py @@ -5,6 +5,7 @@ from datetime import datetime from random import randint +from samples.concepts.audio.utils import check_audio_devices from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai import ( @@ -13,7 +14,7 @@ OpenAIRealtimeExecutionSettings, TurnDetection, ) -from semantic_kernel.connectors.ai.utils import SKAudioPlayer +from semantic_kernel.connectors.ai.utils import SKAudioPlayer, SKAudioTrack from semantic_kernel.contents import ChatHistory from semantic_kernel.functions import kernel_function @@ -43,12 +44,6 @@ # you can check the available devices by uncommenting line below the function -def check_audio_devices(): - import sounddevice as sd - - logger.debug(sd.query_devices()) - - check_audio_devices() @@ -75,11 +70,18 @@ async def main() -> None: kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) kernel.add_function(plugin_name="time", function_name="get_date_time", function=get_date_time) - # create the realtime client and optionally add the audio output function, this is optional + # create the audio player and audio track + # both take a device_id parameter, which is the index of the device to use, if None the default device is used audio_player = SKAudioPlayer() + audio_track = SKAudioTrack() + # create the realtime client and optionally add the audio output function, this is optional # you can define the protocol to use, either "websocket" or "webrtc" # they will behave the same way, even though the underlying protocol is quite different - realtime_client = OpenAIRealtime(protocol="webrtc", audio_output_callback=audio_player.client_callback) + realtime_client = OpenAIRealtime( + protocol="webrtc", + audio_output_callback=audio_player.client_callback, + audio_track=audio_track, + ) # Create the settings for the session # The realtime api, does not use a system message, but takes instructions as a parameter for a session @@ -112,18 +114,9 @@ async def main() -> None: await realtime_client.update_session( settings=settings, chat_history=chat_history, kernel=kernel, create_response=True ) - # you can also send other events to the service, like this (the first has content, the second does not) - # await realtime_client.send( - # SendEvents.CONVERSATION_ITEM_CREATE, - # item=ChatMessageContent(role="user", content="Hi there, who are you?")}, - # ) - # await realtime_client.send(SendEvents.RESPONSE_CREATE) print("Mosscap (transcript): ", end="") - async for event in realtime_client.start_streaming(): + async for event in realtime_client.receive(): match event.event_type: - # case "audio": - # if play_audio and audio_player: - # await audio_player.add_audio(event.audio) case "text": if print_transcript: print(event.text.text, end="") @@ -135,8 +128,6 @@ async def main() -> None: print("") case ListenEvents.ERROR: logger.error(event.event) - # case ....: - # # add other event handling here if __name__ == "__main__": diff --git a/python/samples/concepts/audio/utils.py b/python/samples/concepts/audio/utils.py new file mode 100644 index 000000000000..fda9ecb7d772 --- /dev/null +++ b/python/samples/concepts/audio/utils.py @@ -0,0 +1,11 @@ +# Copyright (c) Microsoft. All rights reserved. + +import logging + +import sounddevice as sd + +logger = logging.getLogger(__name__) + + +def check_audio_devices(): + logger.debug(sd.query_devices()) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index 1a7c5acc330d..af0d1bd8b8bd 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,7 +1,7 @@ # Copyright (c) Microsoft. All rights reserved. from collections.abc import Callable, Coroutine, Mapping -from typing import Any, ClassVar, Literal, TypeVar +from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar from numpy import ndarray from openai import AsyncOpenAI @@ -17,6 +17,9 @@ from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError +if TYPE_CHECKING: + from aiortc.mediastreams import MediaStreamTrack + _T = TypeVar("_T", bound="OpenAIRealtime") @@ -33,6 +36,7 @@ def __init__( self, protocol: Literal["websocket", "webrtc"] = "websocket", audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None, + audio_track: "MediaStreamTrack | None" = None, ai_model_id: str | None = None, api_key: str | None = None, org_id: str | None = None, @@ -54,6 +58,9 @@ def __init__( It is called first in both websockets and webrtc. Even when passed, the audio content will still be added to the receiving queue. + audio_track: The audio track to use for the service, only used by WebRTC. + A default is supplied if not provided. + It can be any class that implements the AudioStreamTrack interface. ai_model_id (str | None): OpenAI model name, see https://platform.openai.com/docs/models service_id (str | None): Service ID tied to the execution settings. @@ -81,6 +88,7 @@ def __init__( raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.realtime_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") + kwargs = {"audio_track": audio_track} if protocol == "webrtc" and audio_track else {} super().__init__( protocol=protocol, audio_output_callback=audio_output_callback, @@ -91,6 +99,7 @@ def __init__( ai_model_type=OpenAIModelTypes.REALTIME, default_headers=default_headers, client=async_client, + **kwargs, ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py index 5f2b49020fb6..f7344b5262ee 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py @@ -1,28 +1,22 @@ # Copyright (c) Microsoft. All rights reserved. +import base64 +import json import logging import sys +from abc import abstractmethod from collections.abc import AsyncGenerator, Callable, Coroutine from typing import TYPE_CHECKING, Any, ClassVar, Literal -from semantic_kernel.contents.function_result_content import FunctionResultContent -from semantic_kernel.contents.realtime_event import ( - FunctionCallEvent, - FunctionResultEvent, - RealtimeEvent, - ServiceEvent, - TextEvent, -) -from semantic_kernel.contents.streaming_text_content import StreamingTextContent - if sys.version_info >= (3, 12): from typing import override # pragma: no cover else: from typing_extensions import override # pragma: no cover from numpy import ndarray -from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent -from openai.types.beta.realtime.response_function_call_arguments_done_event import ( +from openai.types.beta.realtime import ( + RealtimeClientEvent, + RealtimeServerEvent, ResponseFunctionCallArgumentsDoneEvent, ) from pydantic import PrivateAttr @@ -35,12 +29,24 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents from semantic_kernel.connectors.ai.open_ai.services.realtime.utils import ( + _create_realtime_client_event, update_settings_from_function_call_configuration, ) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.chat_message_content import ChatMessageContent +from semantic_kernel.contents.events.realtime_event import ( + FunctionCallEvent, + FunctionResultEvent, + RealtimeEvent, + ServiceEvent, + TextEvent, +) from semantic_kernel.contents.function_call_content import FunctionCallContent +from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.streaming_text_content import StreamingTextContent +from semantic_kernel.contents.text_content import TextContent from semantic_kernel.kernel import Kernel from semantic_kernel.utils.experimental_decorator import experimental_class @@ -61,7 +67,7 @@ class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None kernel: Kernel | None = None - _current_settings: PromptExecutionSettings | None = PrivateAttr(None) + _current_settings: PromptExecutionSettings | None = PrivateAttr(default=None) _call_id_to_function_map: dict[str, str] = PrivateAttr(default_factory=dict) async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[RealtimeEvent, None]: @@ -132,7 +138,11 @@ async def update_session( kernel=self.kernel, # type: ignore ) await self.send( - ServiceEvent(event_type="service", service_type=SendEvents.SESSION_UPDATE, event=self._current_settings) + ServiceEvent( + event_type="service", + service_type=SendEvents.SESSION_UPDATE, + event={"settings": self._current_settings}, + ) ) if chat_history and len(chat_history) > 0: for msg in chat_history.messages: @@ -198,22 +208,185 @@ async def _parse_function_call_arguments_done( # This allows a user to have a full conversation in his code yield FunctionResultEvent(event_type="function_result", function_result=created_output) - @override - async def start_listening( - self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any - ) -> None: - pass - - @override - async def start_sending(self, **kwargs: Any) -> None: - pass + @abstractmethod + async def _send(self, event: RealtimeClientEvent) -> None: + """Send an event to the service.""" + raise NotImplementedError @override - async def create_session( - self, settings: PromptExecutionSettings | None = None, chat_history: ChatHistory | None = None, **kwargs: Any - ) -> None: - pass + async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: + match event.event_type: + case "audio": + if isinstance(event.audio.data, ndarray): + audio_data = base64.b64encode(event.audio.data.tobytes()).decode("utf-8") + else: + audio_data = event.audio.data.decode("utf-8") + await self._send( + _create_realtime_client_event( + event_type=SendEvents.INPUT_AUDIO_BUFFER_APPEND, + audio=audio_data, + ) + ) + case "text": + await self._send( + _create_realtime_client_event( + event_type=SendEvents.CONVERSATION_ITEM_CREATE, + **dict( + type="message", + content=[ + { + "type": "input_text", + "text": event.text.text, + } + ], + role="user", + ), + ) + ) + case "function_call": + await self._send( + _create_realtime_client_event( + event_type=SendEvents.CONVERSATION_ITEM_CREATE, + **dict( + type="function_call", + name=event.function_call.name or event.function_call.function_name, + arguments="" + if not event.function_call.arguments + else event.function_call.arguments + if isinstance(event.function_call.arguments, str) + else json.dumps(event.function_call.arguments), + call_id=event.function_call.metadata.get("call_id"), + ), + ) + ) + case "function_result": + await self._send( + _create_realtime_client_event( + event_type=SendEvents.CONVERSATION_ITEM_CREATE, + **dict( + type="function_call_output", + output=event.function_result.result, + call_id=event.function_result.metadata.get("call_id"), + ), + ) + ) + case "service": + data = event.event + match event.service_type: + case SendEvents.SESSION_UPDATE: + if not data: + logger.error("Event data is empty") + return + settings = data.get("settings", None) + if not settings or not isinstance(settings, PromptExecutionSettings): + logger.error("Event data does not contain 'settings'") + return + if not settings.ai_model_id: + settings.ai_model_id = self.ai_model_id + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + **settings.prepare_settings_dict(), + ) + ) + case SendEvents.INPUT_AUDIO_BUFFER_APPEND: + if not data or "audio" not in data: + logger.error("Event data does not contain 'audio'") + return + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + audio=data["audio"], + ) + ) + case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: + await self._send(_create_realtime_client_event(event_type=event.service_type)) + case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: + await self._send(_create_realtime_client_event(event_type=event.service_type)) + case SendEvents.CONVERSATION_ITEM_CREATE: + if not data or "item" not in data: + logger.error("Event data does not contain 'item'") + return + content = data["item"] + contents = content.items if isinstance(content, ChatMessageContent) else [content] + for item in contents: + match item: + case TextContent(): + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + **dict( + type="message", + content=[ + { + "type": "input_text", + "text": item.text, + } + ], + role="user", + ), + ) + ) + case FunctionCallContent(): + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + **dict( + type="function_call", + name=item.name or item.function_name, + arguments="" + if not item.arguments + else item.arguments + if isinstance(item.arguments, str) + else json.dumps(item.arguments), + call_id=item.metadata.get("call_id"), + ), + ) + ) - @override - async def close_session(self) -> None: - pass + case FunctionResultContent(): + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + **dict( + type="function_call_output", + output=item.result, + call_id=item.metadata.get("call_id"), + ), + ) + ) + case SendEvents.CONVERSATION_ITEM_TRUNCATE: + if not data or "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + item_id=data["item_id"], + content_index=0, + audio_end_ms=data.get("audio_end_ms", 0), + ) + ) + case SendEvents.CONVERSATION_ITEM_DELETE: + if not data or "item_id" not in data: + logger.error("Event data does not contain 'item_id'") + return + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + item_id=data["item_id"], + ) + ) + case SendEvents.RESPONSE_CREATE: + await self._send( + _create_realtime_client_event( + event_type=event.service_type, event_id=data.get("event_id", None) if data else None + ) + ) + case SendEvents.RESPONSE_CANCEL: + await self._send( + _create_realtime_client_event( + event_type=event.service_type, + response_id=data.get("response_id", None) if data else None, + ) + ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py index 1225c2927345..11d8676f45b6 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py @@ -8,8 +8,6 @@ from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, ClassVar, Literal, cast -from pydantic import Field - if sys.version_info >= (3, 12): from typing import override # pragma: no cover else: @@ -17,6 +15,7 @@ from aiohttp import ClientSession from aiortc import ( + MediaStreamTrack, RTCConfiguration, RTCDataChannel, RTCIceServer, @@ -25,22 +24,19 @@ ) from av.audio.frame import AudioFrame from openai._models import construct_type_unchecked -from openai.types.beta.realtime.conversation_item_param import ConversationItemParam +from openai.types.beta.realtime.realtime_client_event import RealtimeClientEvent from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent +from pydantic import Field, PrivateAttr -from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase from semantic_kernel.connectors.ai.realtime_client_base import RealtimeEvent +from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioTrack from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.function_call_content import FunctionCallContent -from semantic_kernel.contents.function_result_content import FunctionResultContent -from semantic_kernel.contents.realtime_event import AudioEvent -from semantic_kernel.contents.text_content import TextContent +from semantic_kernel.contents.events.realtime_event import AudioEvent from semantic_kernel.utils.experimental_decorator import experimental_class if TYPE_CHECKING: - from aiortc import MediaStreamTrack - from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings from semantic_kernel.contents.chat_history import ChatHistory @@ -55,137 +51,50 @@ class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase): protocol: ClassVar[Literal["webrtc"]] = "webrtc" peer_connection: RTCPeerConnection | None = None data_channel: RTCDataChannel | None = None - receive_buffer: asyncio.Queue[RealtimeEvent] = Field(default_factory=asyncio.Queue) - - # region public methods + audio_track: MediaStreamTrack = Field(default_factory=SKAudioTrack) + _receive_buffer: asyncio.Queue[RealtimeEvent] = PrivateAttr(default_factory=asyncio.Queue) @override - async def start_listening( + async def receive( self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, - create_response: bool = False, **kwargs: Any, ) -> AsyncGenerator[RealtimeEvent, None]: - if chat_history or settings or create_response: - await self.update_session(settings=settings, chat_history=chat_history, create_response=create_response) while True: - event = await self.receive_buffer.get() + event = await self._receive_buffer.get() yield event - @override - async def start_sending(self, **kwargs: Any) -> None: + async def _send(self, event: RealtimeClientEvent) -> None: if not self.data_channel: logger.error("Data channel not initialized") return while self.data_channel.readyState != "open": await asyncio.sleep(0.1) - while True: - event = await self.send_buffer.get() - response: dict[str, Any] = {"type": event.event_type} - match event.event_type: - case SendEvents.SESSION_UPDATE: - if "settings" not in event.data: - logger.error("Event data does not contain 'settings'") - response["session"] = event.data["settings"].prepare_settings_dict() - case SendEvents.CONVERSATION_ITEM_CREATE: - if "item" not in event.data: - logger.error("Event data does not contain 'item'") - return - content = event.data["item"] - for item in content.items: - match item: - case TextContent(): - response["item"] = ConversationItemParam( - type="message", - content=[ - { - "type": "input_text", - "text": item.text, - } - ], - role="user", - ) - - case FunctionCallContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function call needs to have a call_id") - continue - response["item"] = ConversationItemParam( - type="function_call", - name=item.name or item.function_name, - arguments="" - if not item.arguments - else item.arguments - if isinstance(item.arguments, str) - else json.dumps(item.arguments), - call_id=call_id, - ) - - case FunctionResultContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function result needs to have a call_id") - continue - response["item"] = ConversationItemParam( - type="function_call_output", - output=item.result, - call_id=call_id, - ) - - case SendEvents.CONVERSATION_ITEM_TRUNCATE: - if "item_id" not in event.data: - logger.error("Event data does not contain 'item_id'") - return - response["item_id"] = event.data["item_id"] - response["content_index"] = 0 - response["audio_end_ms"] = event.data.get("audio_end_ms", 0) - - case SendEvents.CONVERSATION_ITEM_DELETE: - if "item_id" not in event.data: - logger.error("Event data does not contain 'item_id'") - return - response["item_id"] = event.data["item_id"] - case SendEvents.RESPONSE_CREATE: - if "response" in event.data: - response["response"] = event.data["response"] - case SendEvents.RESPONSE_CANCEL: - if "response_id" in event.data: - response["response_id"] = event.data["response_id"] - - try: - self.data_channel.send(json.dumps(response)) - except Exception as e: - logger.error(f"Failed to send event {event} with error: {e!s}") + try: + self.data_channel.send(event.model_dump_json(exclude_none=True)) + except Exception as e: + logger.error(f"Failed to send event {event} with error: {e!s}") @override async def create_session( self, settings: "PromptExecutionSettings | None" = None, chat_history: "ChatHistory | None" = None, - audio_track: "MediaStreamTrack | None" = None, **kwargs: Any, ) -> None: """Create a session in the service.""" - if not audio_track: - from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioTrack - - audio_track = SKAudioTrack() - self.peer_connection = RTCPeerConnection( configuration=RTCConfiguration(iceServers=[RTCIceServer(urls="stun:stun.l.google.com:19302")]) ) # track is the audio track being returned from the service - self.peer_connection.on("track")(self._on_track) + self.peer_connection.add_listener("track", self._on_track) # data channel is used to send and receive messages self.data_channel = self.peer_connection.createDataChannel("oai-events", protocol="json") - self.data_channel.on("message")(self._on_data) + self.data_channel.add_listener("message", self._on_data) # this is the incoming audio, which sends audio to the service - self.peer_connection.addTransceiver(audio_track) + self.peer_connection.addTransceiver(self.audio_track) offer = await self.peer_connection.createOffer() await self.peer_connection.setLocalDescription(offer) @@ -230,8 +139,6 @@ async def close_session(self) -> None: self.data_channel.close() self.data_channel = None - # region implementation specifics - async def _on_track(self, track: "MediaStreamTrack") -> None: logger.info(f"Received {track.kind} track from remote") if track.kind != "audio": @@ -252,7 +159,7 @@ async def _on_track(self, track: "MediaStreamTrack") -> None: except Exception as e: logger.error(f"Error playing remote audio frame: {e!s}") try: - await self.receive_buffer.put( + await self._receive_buffer.put( AudioEvent( event_type="audio", service_type=ListenEvents.RESPONSE_AUDIO_DELTA, @@ -278,7 +185,7 @@ async def _on_data(self, data: str) -> None: logger.error(f"Failed to parse event {data} with error: {e!s}") return async for parsed_event in self._parse_event(event): - await self.receive_buffer.put(parsed_event) + await self._receive_buffer.put(parsed_event) async def _get_ephemeral_token(self) -> str: """Get an ephemeral token from OpenAI.""" diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py index 4f32067ba3cc..db2d0cfea51d 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py @@ -2,30 +2,26 @@ import asyncio import base64 -import json import logging import sys from collections.abc import AsyncGenerator from typing import TYPE_CHECKING, Any, ClassVar, Literal -import numpy as np - if sys.version_info >= (3, 12): from typing import override # pragma: no cover else: from typing_extensions import override # pragma: no cover +import numpy as np from openai.resources.beta.realtime.realtime import AsyncRealtimeConnection -from openai.types.beta.realtime.conversation_item_param import ConversationItemParam +from openai.types.beta.realtime.realtime_client_event import RealtimeClientEvent from pydantic import Field -from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.function_call_content import FunctionCallContent -from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.events.realtime_event import RealtimeEvent from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent -from semantic_kernel.contents.text_content import TextContent from semantic_kernel.contents.utils.author_role import AuthorRole from semantic_kernel.utils.experimental_decorator import experimental_class @@ -35,8 +31,6 @@ logger: logging.Logger = logging.getLogger(__name__) -# region Websocket - @experimental_class class OpenAIRealtimeWebsocketBase(OpenAIRealtimeBase): @@ -47,20 +41,14 @@ class OpenAIRealtimeWebsocketBase(OpenAIRealtimeBase): connected: asyncio.Event = Field(default_factory=asyncio.Event) @override - async def start_listening( + async def receive( self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, - create_response: bool = False, **kwargs: Any, - ) -> AsyncGenerator[tuple[str, Any], None]: + ) -> AsyncGenerator[RealtimeEvent, None]: await self.connected.wait() if not self.connection: raise ValueError("Connection is not established.") - if chat_history or settings or create_response: - await self.update_session(settings=settings, chat_history=chat_history, create_response=create_response) - async for event in self.connection: if event.type == ListenEvents.RESPONSE_AUDIO_DELTA.value: if self.audio_output_callback: @@ -86,98 +74,14 @@ async def start_listening( async for event in self._parse_event(event): yield event - @override - async def start_sending(self, **kwargs: Any) -> None: + async def _send(self, event: RealtimeClientEvent) -> None: await self.connected.wait() if not self.connection: raise ValueError("Connection is not established.") - while True: - event = await self.send_buffer.get() - match event.event_type: - case SendEvents.SESSION_UPDATE: - if "settings" not in event.data: - logger.error("Event data does not contain 'settings'") - await self.connection.session.update(session=event.data["settings"].prepare_settings_dict()) - case SendEvents.INPUT_AUDIO_BUFFER_APPEND: - if "content" not in event.data: - logger.error("Event data does not contain 'content'") - return - await self.connection.input_audio_buffer.append(audio=event.data["content"].data.decode("utf-8")) - case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: - await self.connection.input_audio_buffer.commit() - case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: - await self.connection.input_audio_buffer.clear() - case SendEvents.CONVERSATION_ITEM_CREATE: - if "item" not in event.data: - logger.error("Event data does not contain 'item'") - return - content = event.data["item"] - for item in content.items: - match item: - case TextContent(): - await self.connection.conversation.item.create( - item=ConversationItemParam( - type="message", - content=[ - { - "type": "input_text", - "text": item.text, - } - ], - role="user", - ) - ) - case FunctionCallContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function call needs to have a call_id") - continue - await self.connection.conversation.item.create( - item=ConversationItemParam( - type="function_call", - name=item.name or item.function_name, - arguments="" - if not item.arguments - else item.arguments - if isinstance(item.arguments, str) - else json.dumps(item.arguments), - call_id=call_id, - ) - ) - case FunctionResultContent(): - call_id = item.metadata.get("call_id") - if not call_id: - logger.error("Function result needs to have a call_id") - continue - await self.connection.conversation.item.create( - item=ConversationItemParam( - type="function_call_output", - output=item.result, - call_id=call_id, - ) - ) - case SendEvents.CONVERSATION_ITEM_TRUNCATE: - if "item_id" not in event.data: - logger.error("Event data does not contain 'item_id'") - return - await self.connection.conversation.item.truncate( - item_id=event.data["item_id"], content_index=0, audio_end_ms=event.data.get("audio_end_ms", 0) - ) - case SendEvents.CONVERSATION_ITEM_DELETE: - if "item_id" not in event.data: - logger.error("Event data does not contain 'item_id'") - return - await self.connection.conversation.item.delete(item_id=event.data["item_id"]) - case SendEvents.RESPONSE_CREATE: - if "response" in event.data: - await self.connection.response.create(response=event.data["response"]) - else: - await self.connection.response.create() - case SendEvents.RESPONSE_CANCEL: - if "response_id" in event.data: - await self.connection.response.cancel(response_id=event.data["response_id"]) - else: - await self.connection.response.cancel() + try: + await self.connection.send(event) + except Exception as e: + logger.error(f"Error sending response: {e!s}") @override async def create_session( diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py index ada8d42924c0..9aa061e44bc5 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py @@ -2,6 +2,24 @@ from typing import TYPE_CHECKING, Any +from openai.types.beta.realtime import ( + ConversationItem, + ConversationItemCreateEvent, + ConversationItemDeleteEvent, + ConversationItemTruncateEvent, + InputAudioBufferAppendEvent, + InputAudioBufferClearEvent, + InputAudioBufferCommitEvent, + RealtimeClientEvent, + ResponseCancelEvent, + ResponseCreateEvent, + SessionUpdateEvent, +) +from openai.types.beta.realtime.response_create_event import Response +from openai.types.beta.realtime.session_update_event import Session + +from semantic_kernel.connectors.ai.open_ai.services.realtime.const import SendEvents + if TYPE_CHECKING: from semantic_kernel.connectors.ai.function_choice_behavior import ( FunctionCallChoiceConfiguration, @@ -45,3 +63,65 @@ def kernel_function_metadata_to_function_call_format( "required": [p.name for p in metadata.parameters if p.is_required and p.include_in_function_choices], }, } + + +def _create_realtime_client_event(event_type: SendEvents, **kwargs: Any) -> RealtimeClientEvent: + match event_type: + case SendEvents.SESSION_UPDATE: + event_kwargs = {"event_id": kwargs.pop("event_id")} if "event_id" in kwargs else {} + return SessionUpdateEvent( + type=event_type, + session=Session.model_validate(kwargs), + **event_kwargs, + ) + case SendEvents.INPUT_AUDIO_BUFFER_APPEND: + return InputAudioBufferAppendEvent( + type=event_type, + **kwargs, + ) + case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: + return InputAudioBufferCommitEvent( + type=event_type, + **kwargs, + ) + case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: + return InputAudioBufferClearEvent( + type=event_type, + **kwargs, + ) + case SendEvents.CONVERSATION_ITEM_CREATE: + if "event_id" in kwargs: + event_id = kwargs.pop("event_id") + if "previous_item_id" in kwargs: + previous_item_id = kwargs.pop("previous_item_id") + event_kwargs = {"event_id": event_id} if "event_id" in kwargs else {} + event_kwargs.update({"previous_item_id": previous_item_id} if "previous_item_id" in kwargs else {}) + return ConversationItemCreateEvent( + type=event_type, + item=ConversationItem.model_validate(kwargs), + **event_kwargs, + ) + case SendEvents.CONVERSATION_ITEM_TRUNCATE: + return ConversationItemTruncateEvent( + type=event_type, + **kwargs, + ) + case SendEvents.CONVERSATION_ITEM_DELETE: + return ConversationItemDeleteEvent( + type=event_type, + **kwargs, + ) + case SendEvents.RESPONSE_CREATE: + event_kwargs = {"event_id": kwargs.pop("event_id")} if "event_id" in kwargs else {} + return ResponseCreateEvent( + type=event_type, + response=Response.model_validate(kwargs), + **event_kwargs, + ) + case SendEvents.RESPONSE_CANCEL: + return ResponseCancelEvent( + type=event_type, + **kwargs, + ) + case _: + raise ValueError(f"Unknown event type: {event_type}") diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index a6a332791293..0ad1fc13a089 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -2,20 +2,17 @@ import sys from abc import ABC, abstractmethod -from asyncio import Queue from collections.abc import AsyncGenerator, Callable from typing import TYPE_CHECKING, Any, ClassVar -from pydantic import Field - if sys.version_info >= (3, 11): - from asyncio import TaskGroup + from typing import Self # pragma: no cover else: - from taskgroup import TaskGroup + from typing_extensions import Self # pragma: no cover from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType -from semantic_kernel.contents.realtime_event import RealtimeEvent +from semantic_kernel.contents.events.realtime_event import RealtimeEvent from semantic_kernel.services.ai_service_client_base import AIServiceClientBase from semantic_kernel.utils.experimental_decorator import experimental_class @@ -29,8 +26,8 @@ class RealtimeClientBase(AIServiceClientBase, ABC): """Base class for a realtime client.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False - send_buffer: Queue[RealtimeEvent] = Field(default_factory=Queue) + @abstractmethod async def send(self, event: RealtimeEvent) -> None: """Send an event to the service. @@ -38,53 +35,20 @@ async def send(self, event: RealtimeEvent) -> None: event: The event to send. kwargs: Additional arguments. """ - await self.send_buffer.put(event) - - async def start_streaming( - self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, - **kwargs: Any, - ) -> AsyncGenerator[RealtimeEvent, None]: - """Start streaming, will start both listening and sending. - - This method, start tasks for both listening and sending. - - The arguments are passed to the start_listening method. - - Args: - settings: Prompt execution settings. - chat_history: Chat history. - kwargs: Additional arguments. - """ - await self.update_session(settings=settings, chat_history=chat_history, **kwargs) - async with TaskGroup() as tg: - tg.create_task(self.start_sending(**kwargs)) - yield from tg.create_task(self.start_listening()) + raise NotImplementedError @abstractmethod - async def start_listening( + def receive( self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, **kwargs: Any, ) -> AsyncGenerator[RealtimeEvent, None]: """Starts listening for messages from the service, generates events. Args: - settings: Prompt execution settings. - chat_history: Chat history. kwargs: Additional arguments. """ raise NotImplementedError - @abstractmethod - async def start_sending( - self, - ) -> None: - """Start sending items from the input_buffer to the service.""" - raise NotImplementedError - @abstractmethod async def create_session( self, @@ -134,7 +98,7 @@ def _update_function_choice_settings_callback( """ return lambda configuration, settings, choice_type: None - async def __aenter__(self) -> "RealtimeClientBase": + async def __aenter__(self) -> "Self": """Enter the context manager. Default implementation calls the create session method. diff --git a/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py index dd6d0e5fe16f..ed2ef294c716 100644 --- a/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py +++ b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py @@ -2,14 +2,14 @@ import asyncio import logging -from typing import Any, Final +from typing import Any, ClassVar, Final import numpy as np import numpy.typing as npt from aiortc.mediastreams import MediaStreamError, MediaStreamTrack from av.audio.frame import AudioFrame from av.frame import Frame -from pydantic import Field, PrivateAttr +from pydantic import PrivateAttr from sounddevice import InputStream, OutputStream from semantic_kernel.contents.audio_content import AudioContent @@ -25,36 +25,54 @@ class SKAudioTrack(KernelBaseModel, MediaStreamTrack): - """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. + """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice.""" - Make sure the device_id is set to the correct device for your system. - """ - - kind: str = "audio" + kind: ClassVar[str] = "audio" + device_id: str | int | None = None sample_rate: int = SAMPLE_RATE channels: int = TRACK_CHANNELS frame_duration: int = FRAME_DURATION dtype: npt.DTypeLike = DTYPE - device: str | int | None = None - queue: asyncio.Queue[Frame] = Field(default_factory=asyncio.Queue) - is_recording: bool = False frame_size: int = 0 + _queue: asyncio.Queue[Frame] = PrivateAttr(default_factory=asyncio.Queue) + _is_recording: bool = False _stream: InputStream | None = None _recording_task: asyncio.Task | None = None _loop: asyncio.AbstractEventLoop | None = None - _pts: int = 0 # Add this to track the pts + _pts: int = 0 - def __init__(self, **kwargs: Any): - """Initialize the audio track. + def __init__( + self, + *, + device_id: str | int | None = None, + sample_rate: int = SAMPLE_RATE, + channels: int = TRACK_CHANNELS, + frame_duration: int = FRAME_DURATION, + dtype: npt.DTypeLike = DTYPE, + ): + """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. + + Make sure the device_id is set to the correct device for your system. Args: + device_id: The device id to use for recording audio. + sample_rate: The sample rate for the audio. + channels: The number of channels for the audio. + frame_duration: The duration of each audio frame in milliseconds. + dtype: The data type for the audio. **kwargs: Additional keyword arguments. - """ - kwargs["frame_size"] = int( - kwargs.get("sample_rate", SAMPLE_RATE) * kwargs.get("frame_duration", FRAME_DURATION) / 1000 + args = { + "device_id": device_id, + "sample_rate": sample_rate, + "channels": channels, + "frame_duration": frame_duration, + "dtype": dtype, + } + args["frame_size"] = int( + args.get("sample_rate", SAMPLE_RATE) * args.get("frame_duration", FRAME_DURATION) / 1000 ) - super().__init__(**kwargs) + super().__init__(**args) MediaStreamTrack.__init__(self) async def recv(self) -> Frame: @@ -63,8 +81,8 @@ async def recv(self) -> Frame: self._recording_task = asyncio.create_task(self.start_recording()) try: - frame = await self.queue.get() - self.queue.task_done() + frame = await self._queue.get() + self._queue.task_done() return frame except Exception as e: logger.error(f"Error receiving audio frame: {e!s}") @@ -74,7 +92,7 @@ def _sounddevice_callback(self, indata: np.ndarray, frames: int, time: Any, stat if status: logger.warning(f"Audio input status: {status}") if self._loop and self._loop.is_running(): - asyncio.run_coroutine_threadsafe(self.queue.put(self._create_frame(indata)), self._loop) + asyncio.run_coroutine_threadsafe(self._queue.put(self._create_frame(indata)), self._loop) def _create_frame(self, indata: np.ndarray) -> Frame: audio_data = indata.copy() @@ -95,16 +113,16 @@ def _create_frame(self, indata: np.ndarray) -> Frame: async def start_recording(self): """Start recording audio from the input device.""" - if self.is_recording: + if self._is_recording: return - self.is_recording = True + self._is_recording = True self._loop = asyncio.get_running_loop() self._pts = 0 # Reset pts when starting recording try: self._stream = InputStream( - device=self.device, + device=self.device_id, channels=self.channels, samplerate=self.sample_rate, dtype=self.dtype, @@ -113,14 +131,14 @@ async def start_recording(self): ) self._stream.start() - while self.is_recording: + while self._is_recording: await asyncio.sleep(0.1) except Exception as e: logger.error(f"Error in audio recording: {e!s}") raise finally: - self.is_recording = False + self._is_recording = False class SKAudioPlayer(KernelBaseModel): @@ -128,17 +146,26 @@ class SKAudioPlayer(KernelBaseModel): Make sure the device_id is set to the correct device for your system. - The sample rate, channels and frame duration should be set to match the audio you - are receiving, the defaults are for WebRTC. + The sample rate, channels and frame duration + should be set to match the audio you + are receiving. + + Args: + device_id: The device id to use for playing audio. + sample_rate: The sample rate for the audio. + channels: The number of channels for the audio. + dtype: The data type for the audio. + frame_duration: The duration of each audio frame in milliseconds + """ device_id: int | None = None sample_rate: int = SAMPLE_RATE - dtype: npt.DTypeLike = DTYPE channels: int = PLAYER_CHANNELS - frame_duration_ms: int = FRAME_DURATION - _queue: asyncio.Queue[np.ndarray] | None = None - _stream: OutputStream | None = PrivateAttr(None) + dtype: npt.DTypeLike = DTYPE + frame_duration: int = FRAME_DURATION + _queue: asyncio.Queue[np.ndarray] | None = PrivateAttr(default=None) + _stream: OutputStream | None = PrivateAttr(default=None) async def __aenter__(self): """Start the audio stream when entering a context.""" @@ -157,7 +184,7 @@ def start(self): samplerate=self.sample_rate, channels=self.channels, dtype=self.dtype, - blocksize=int(self.sample_rate * self.frame_duration_ms / 1000), + blocksize=int(self.sample_rate * self.frame_duration / 1000), device=self.device_id, ) if self._stream and self._queue: diff --git a/python/semantic_kernel/contents/events/__init__.py b/python/semantic_kernel/contents/events/__init__.py new file mode 100644 index 000000000000..7466a652364b --- /dev/null +++ b/python/semantic_kernel/contents/events/__init__.py @@ -0,0 +1,19 @@ +# Copyright (c) Microsoft. All rights reserved. + +from semantic_kernel.contents.events.realtime_event import ( + AudioEvent, + FunctionCallEvent, + FunctionResultEvent, + RealtimeEvent, + ServiceEvent, + TextEvent, +) + +__all__ = [ + "AudioEvent", + "FunctionCallEvent", + "FunctionResultEvent", + "RealtimeEvent", + "ServiceEvent", + "TextEvent", +] diff --git a/python/semantic_kernel/contents/realtime_event.py b/python/semantic_kernel/contents/events/realtime_event.py similarity index 100% rename from python/semantic_kernel/contents/realtime_event.py rename to python/semantic_kernel/contents/events/realtime_event.py From cf185969477191d9017d8c5d003e239dde66e995 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Thu, 30 Jan 2025 19:59:53 +0100 Subject: [PATCH 23/25] WIP --- .../audio/04-chat_with_realtime_api_simple.py | 2 +- .../05-chat_with_realtime_api_complex.py | 22 +++-- .../ai/open_ai/services/open_ai_realtime.py | 84 ++++++++++++++++--- .../realtime/open_ai_realtime_base.py | 49 ++++++----- .../realtime/open_ai_realtime_webrtc.py | 3 +- .../realtime/open_ai_realtime_websocket.py | 21 ++--- .../ai/open_ai/services/realtime/utils.py | 5 +- .../connectors/ai/utils/realtime_helpers.py | 44 +++++++--- .../contents/binary_content.py | 11 +++ .../contents/events/realtime_event.py | 10 +-- 10 files changed, 171 insertions(+), 80 deletions(-) diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py b/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py index 5dda1dc6d308..06ee11807a81 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py +++ b/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py @@ -43,7 +43,7 @@ async def main() -> None: # create the realtime client and optionally add the audio output function, this is optional # you can define the protocol to use, either "websocket" or "webrtc" # they will behave the same way, even though the underlying protocol is quite different - realtime_client = OpenAIRealtime(protocol="webrtc") + realtime_client = OpenAIRealtime("webrtc") # Create the settings for the session settings = OpenAIRealtimeExecutionSettings( instructions=""" diff --git a/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py b/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py index 77406af4f355..b567c0028178 100644 --- a/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py +++ b/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py @@ -52,35 +52,41 @@ def get_weather(location: str) -> str: """Get the weather for a location.""" weather_conditions = ("sunny", "hot", "cloudy", "raining", "freezing", "snowing") weather = weather_conditions[randint(0, len(weather_conditions) - 1)] # nosec - logger.info(f"Getting weather for {location}: {weather}") + logger.info(f"@ Getting weather for {location}: {weather}") return f"The weather in {location} is {weather}." @kernel_function def get_date_time() -> str: """Get the current date and time.""" - logger.info("Getting current datetime") + logger.info("@ Getting current datetime") return f"The current date and time is {datetime.now().isoformat()}." +@kernel_function +def goodbye(): + """When the user is done, say goodbye and then call this function.""" + logger.info("@ Goodbye has been called!") + raise KeyboardInterrupt + + async def main() -> None: print_transcript = True # create the Kernel and add a simple function for function calling. kernel = Kernel() - kernel.add_function(plugin_name="weather", function_name="get_weather", function=get_weather) - kernel.add_function(plugin_name="time", function_name="get_date_time", function=get_date_time) + kernel.add_functions(plugin_name="helpers", functions=[goodbye, get_weather, get_date_time]) # create the audio player and audio track # both take a device_id parameter, which is the index of the device to use, if None the default device is used - audio_player = SKAudioPlayer() + audio_player = SKAudioPlayer(sample_rate=24000, frame_duration=100, channels=1) audio_track = SKAudioTrack() # create the realtime client and optionally add the audio output function, this is optional # you can define the protocol to use, either "websocket" or "webrtc" # they will behave the same way, even though the underlying protocol is quite different realtime_client = OpenAIRealtime( - protocol="webrtc", + protocol="websocket", audio_output_callback=audio_player.client_callback, - audio_track=audio_track, + # audio_track=audio_track, ) # Create the settings for the session @@ -110,7 +116,7 @@ async def main() -> None: chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") # the context manager calls the create_session method on the client and start listening to the audio stream - async with realtime_client, audio_player: + async with realtime_client, audio_player, audio_track.stream_to_realtime_client(realtime_client): await realtime_client.update_session( settings=settings, chat_history=chat_history, kernel=kernel, create_response=True ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index af0d1bd8b8bd..b9e809d4e396 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. -from collections.abc import Callable, Coroutine, Mapping +from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar from numpy import ndarray @@ -15,26 +15,66 @@ OpenAIRealtimeWebsocketBase, ) from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings +from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase +from semantic_kernel.contents.chat_history import ChatHistory +from semantic_kernel.contents.events.realtime_event import RealtimeEvent from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError if TYPE_CHECKING: from aiortc.mediastreams import MediaStreamTrack + from semantic_kernel.connectors.ai import PromptExecutionSettings + from semantic_kernel.contents import ChatHistory + _T = TypeVar("_T", bound="OpenAIRealtime") -class OpenAIRealtime(OpenAIConfigBase, OpenAIRealtimeBase): +__all__ = ["OpenAIRealtime"] + + +class RealtimeClientStub(RealtimeClientBase): + """This class makes sure that IDE's don't complain about missing methods in the below superclass.""" + + async def send(self, event: Any) -> None: + pass + + async def create_session( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + **kwargs: Any, + ) -> None: + pass + + def receive(self, **kwargs: Any) -> AsyncGenerator[RealtimeEvent, None]: + pass + + async def update_session( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + **kwargs: Any, + ) -> None: + pass + + async def close_session(self) -> None: + pass + + +class OpenAIRealtime(OpenAIRealtimeBase, RealtimeClientStub): """OpenAI Realtime service.""" - def __new__(cls: type["_T"], *args: Any, **kwargs: Any) -> "_T": + def __new__(cls: type["_T"], protocol: str, *args: Any, **kwargs: Any) -> "_T": """Pick the right subclass, based on protocol.""" subclass_map = {subcl.protocol: subcl for subcl in cls.__subclasses__()} - subclass = subclass_map[kwargs.pop("protocol", "websocket")] + subclass = subclass_map[protocol] return super(OpenAIRealtime, subclass).__new__(subclass) def __init__( self, - protocol: Literal["websocket", "webrtc"] = "websocket", + protocol: Literal["websocket", "webrtc"], + *, audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None, audio_track: "MediaStreamTrack | None" = None, ai_model_id: str | None = None, @@ -42,7 +82,7 @@ def __init__( org_id: str | None = None, service_id: str | None = None, default_headers: Mapping[str, str] | None = None, - async_client: AsyncOpenAI | None = None, + client: AsyncOpenAI | None = None, env_file_path: str | None = None, env_file_encoding: str | None = None, **kwargs: Any, @@ -50,7 +90,7 @@ def __init__( """Initialize an OpenAIRealtime service. Args: - protocol: The protocol to use, can be either "websocket" or "webrtc". + protocol: The protocol to use, must be either "websocket" or "webrtc". audio_output_callback: The audio output callback, optional. This should be a coroutine, that takes a ndarray with audio as input. The goal of this function is to allow you to play the audio with the @@ -70,7 +110,7 @@ def __init__( the env vars or .env file value. default_headers: The default headers mapping of string keys to string values for HTTP requests. (Optional) - async_client (Optional[AsyncOpenAI]): An existing client to use. (Optional) + client (Optional[AsyncOpenAI]): An existing client to use. (Optional) env_file_path (str | None): Use the environment settings file as a fallback to environment variables. (Optional) env_file_encoding (str | None): The encoding of the environment settings file. (Optional) @@ -88,7 +128,6 @@ def __init__( raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.realtime_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") - kwargs = {"audio_track": audio_track} if protocol == "webrtc" and audio_track else {} super().__init__( protocol=protocol, audio_output_callback=audio_output_callback, @@ -98,12 +137,12 @@ def __init__( org_id=openai_settings.org_id, ai_model_type=OpenAIModelTypes.REALTIME, default_headers=default_headers, - client=async_client, + client=client, **kwargs, ) -class OpenAIRealtimeWebRTC(OpenAIRealtime, OpenAIRealtimeWebRTCBase): +class OpenAIRealtimeWebRTC(OpenAIRealtime, OpenAIRealtimeWebRTCBase, OpenAIConfigBase): """OpenAI Realtime service using WebRTC protocol. This should not be used directly, use OpenAIRealtime instead. @@ -112,8 +151,19 @@ class OpenAIRealtimeWebRTC(OpenAIRealtime, OpenAIRealtimeWebRTCBase): protocol: ClassVar[Literal["webrtc"]] = "webrtc" + def __init__( + self, + *args: Any, + **kwargs: Any, + ) -> None: + """Initialize an OpenAIRealtime service using WebRTC protocol.""" + super().__init__( + *args, + **kwargs, + ) + -class OpenAIRealtimeWebSocket(OpenAIRealtime, OpenAIRealtimeWebsocketBase): +class OpenAIRealtimeWebSocket(OpenAIRealtime, OpenAIRealtimeWebsocketBase, OpenAIConfigBase): """OpenAI Realtime service using WebSocket protocol. This should not be used directly, use OpenAIRealtime instead. @@ -121,3 +171,13 @@ class OpenAIRealtimeWebSocket(OpenAIRealtime, OpenAIRealtimeWebsocketBase): """ protocol: ClassVar[Literal["websocket"]] = "websocket" + + def __init__( + self, + *args: Any, + **kwargs: Any, + ) -> None: + super().__init__( + *args, + **kwargs, + ) diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py index f7344b5262ee..0e94dd9c6854 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py @@ -1,10 +1,8 @@ # Copyright (c) Microsoft. All rights reserved. -import base64 import json import logging import sys -from abc import abstractmethod from collections.abc import AsyncGenerator, Callable, Coroutine from typing import TYPE_CHECKING, Any, ClassVar, Literal @@ -146,11 +144,24 @@ async def update_session( ) if chat_history and len(chat_history) > 0: for msg in chat_history.messages: - await self.send( - ServiceEvent(event_type="service", service_type=SendEvents.CONVERSATION_ITEM_CREATE, event=msg) - ) + for item in msg.items: + match item: + case TextContent(): + await self.send(TextEvent(service_type=SendEvents.CONVERSATION_ITEM_CREATE, text=item)) + case FunctionCallContent(): + await self.send( + FunctionCallEvent(service_type=SendEvents.CONVERSATION_ITEM_CREATE, function_call=item) + ) + case FunctionResultContent(): + await self.send( + FunctionResultEvent( + service_type=SendEvents.CONVERSATION_ITEM_CREATE, function_result=item + ) + ) + case _: + logger.error("Unsupported item type: %s", item) if create_response: - await self.send(ServiceEvent(event_type="service", service_type=SendEvents.RESPONSE_CREATE)) + await self.send(ServiceEvent(service_type=SendEvents.RESPONSE_CREATE)) @override def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: @@ -191,24 +202,21 @@ async def _parse_function_call_arguments_done( index=event.output_index, metadata={"call_id": event.call_id}, ) - yield FunctionCallEvent( - event_type="function_call", - service_type=ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, - function_call=item, - ) + yield FunctionCallEvent(service_type=ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DONE, function_call=item) chat_history = ChatHistory() await self.kernel.invoke_function_call(item, chat_history) created_output: FunctionResultContent = chat_history.messages[-1].items[0] # type: ignore # This returns the output to the service - await self.send( - ServiceEvent(event_type="service", service_type=SendEvents.CONVERSATION_ITEM_CREATE, event=created_output) + result = FunctionResultEvent( + service_type=SendEvents.CONVERSATION_ITEM_CREATE, + function_result=created_output, ) + await self.send(result) # The model doesn't start responding to the tool call automatically, so triggering it here. - await self.send(ServiceEvent(event_type="service", service_type=SendEvents.RESPONSE_CREATE)) + await self.send(ServiceEvent(service_type=SendEvents.RESPONSE_CREATE)) # This allows a user to have a full conversation in his code - yield FunctionResultEvent(event_type="function_result", function_result=created_output) + yield result - @abstractmethod async def _send(self, event: RealtimeClientEvent) -> None: """Send an event to the service.""" raise NotImplementedError @@ -217,14 +225,9 @@ async def _send(self, event: RealtimeClientEvent) -> None: async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: match event.event_type: case "audio": - if isinstance(event.audio.data, ndarray): - audio_data = base64.b64encode(event.audio.data.tobytes()).decode("utf-8") - else: - audio_data = event.audio.data.decode("utf-8") await self._send( _create_realtime_client_event( - event_type=SendEvents.INPUT_AUDIO_BUFFER_APPEND, - audio=audio_data, + event_type=SendEvents.INPUT_AUDIO_BUFFER_APPEND, audio=event.audio.to_base64_bytestring() ) ) case "text": @@ -286,7 +289,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: await self._send( _create_realtime_client_event( event_type=event.service_type, - **settings.prepare_settings_dict(), + session=settings.prepare_settings_dict(), ) ) case SendEvents.INPUT_AUDIO_BUFFER_APPEND: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py index 11d8676f45b6..731ff423011b 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py @@ -161,9 +161,8 @@ async def _on_track(self, track: "MediaStreamTrack") -> None: try: await self._receive_buffer.put( AudioEvent( - event_type="audio", service_type=ListenEvents.RESPONSE_AUDIO_DELTA, - audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame), # type: ignore + audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame), ), ) except Exception as e: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py index db2d0cfea51d..8adee40db02d 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py @@ -20,9 +20,7 @@ from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.events.realtime_event import RealtimeEvent -from semantic_kernel.contents.streaming_chat_message_content import StreamingChatMessageContent -from semantic_kernel.contents.utils.author_role import AuthorRole +from semantic_kernel.contents.events.realtime_event import AudioEvent, RealtimeEvent from semantic_kernel.utils.experimental_decorator import experimental_class if TYPE_CHECKING: @@ -54,18 +52,11 @@ async def receive( if self.audio_output_callback: await self.audio_output_callback(np.frombuffer(base64.b64decode(event.delta), dtype=np.int16)) try: - yield ( - event.type, - StreamingChatMessageContent( - role=AuthorRole.ASSISTANT, - items=[ - AudioContent( - data=base64.b64decode(event.delta), - data_format="base64", - inner_content=event, - ) - ], # type: ignore - choice_index=event.content_index, + yield AudioEvent( + audio=AudioContent( + data=base64.b64decode(event.delta), + data_format="base64", + inner_content=event, ), ) except Exception as e: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py index 9aa061e44bc5..a33531ca19c7 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py @@ -68,11 +68,10 @@ def kernel_function_metadata_to_function_call_format( def _create_realtime_client_event(event_type: SendEvents, **kwargs: Any) -> RealtimeClientEvent: match event_type: case SendEvents.SESSION_UPDATE: - event_kwargs = {"event_id": kwargs.pop("event_id")} if "event_id" in kwargs else {} return SessionUpdateEvent( type=event_type, - session=Session.model_validate(kwargs), - **event_kwargs, + session=Session.model_validate(kwargs.pop("session")), + **kwargs, ) case SendEvents.INPUT_AUDIO_BUFFER_APPEND: return InputAudioBufferAppendEvent( diff --git a/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py index ed2ef294c716..33fd09ce7f66 100644 --- a/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py +++ b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py @@ -2,6 +2,7 @@ import asyncio import logging +from contextlib import asynccontextmanager from typing import Any, ClassVar, Final import numpy as np @@ -12,7 +13,9 @@ from pydantic import PrivateAttr from sounddevice import InputStream, OutputStream +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.events.realtime_event import AudioEvent from semantic_kernel.kernel_pydantic import KernelBaseModel logger = logging.getLogger(__name__) @@ -28,7 +31,7 @@ class SKAudioTrack(KernelBaseModel, MediaStreamTrack): """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice.""" kind: ClassVar[str] = "audio" - device_id: str | int | None = None + device: str | int | None = None sample_rate: int = SAMPLE_RATE channels: int = TRACK_CHANNELS frame_duration: int = FRAME_DURATION @@ -44,7 +47,7 @@ class SKAudioTrack(KernelBaseModel, MediaStreamTrack): def __init__( self, *, - device_id: str | int | None = None, + device: str | int | None = None, sample_rate: int = SAMPLE_RATE, channels: int = TRACK_CHANNELS, frame_duration: int = FRAME_DURATION, @@ -52,10 +55,10 @@ def __init__( ): """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. - Make sure the device_id is set to the correct device for your system. + Make sure the device is set to the correct device for your system. Args: - device_id: The device id to use for recording audio. + device: The device id to use for recording audio. sample_rate: The sample rate for the audio. channels: The number of channels for the audio. frame_duration: The duration of each audio frame in milliseconds. @@ -63,7 +66,7 @@ def __init__( **kwargs: Additional keyword arguments. """ args = { - "device_id": device_id, + "device": device, "sample_rate": sample_rate, "channels": channels, "frame_duration": frame_duration, @@ -88,6 +91,15 @@ async def recv(self) -> Frame: logger.error(f"Error receiving audio frame: {e!s}") raise MediaStreamError("Failed to receive audio frame") + @asynccontextmanager + async def stream_to_realtime_client(self, realtime_client: RealtimeClientBase): + """Stream audio data to a RealtimeClientBase.""" + while True: + frame = await self.recv() + await realtime_client.send(AudioEvent(audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16"))) + yield + await asyncio.sleep(0.01) + def _sounddevice_callback(self, indata: np.ndarray, frames: int, time: Any, status: Any) -> None: if status: logger.warning(f"Audio input status: {status}") @@ -122,7 +134,7 @@ async def start_recording(self): try: self._stream = InputStream( - device=self.device_id, + device=self.device, channels=self.channels, samplerate=self.sample_rate, dtype=self.dtype, @@ -151,7 +163,7 @@ class SKAudioPlayer(KernelBaseModel): are receiving. Args: - device_id: The device id to use for playing audio. + device: The device id to use for playing audio. sample_rate: The sample rate for the audio. channels: The number of channels for the audio. dtype: The data type for the audio. @@ -159,7 +171,7 @@ class SKAudioPlayer(KernelBaseModel): """ - device_id: int | None = None + device: int | None = None sample_rate: int = SAMPLE_RATE channels: int = PLAYER_CHANNELS dtype: npt.DTypeLike = DTYPE @@ -185,7 +197,7 @@ def start(self): channels=self.channels, dtype=self.dtype, blocksize=int(self.sample_rate * self.frame_duration / 1000), - device=self.device_id, + device=self.device, ) if self._stream and self._queue: self._stream.start() @@ -205,8 +217,18 @@ def _sounddevice_callback(self, outdata, frames, time, status): if self._queue.empty(): return data: np.ndarray = self._queue.get_nowait() - outdata[:] = data.reshape(outdata.shape) - self._queue.task_done() + if data.size == frames: + outdata[:] = data.reshape(outdata.shape) + self._queue.task_done() + else: + if data.size > frames: + self._queue.put_nowait(data[frames:]) + outdata[:] = np.concatenate((np.empty(0, dtype=np.int16), data[:frames])).reshape(outdata.shape) + else: + outdata[:] = np.concatenate((data, np.zeros(frames - len(data), dtype=np.int16))).reshape( + outdata.shape + ) + self._queue.task_done() async def client_callback(self, content: np.ndarray): """This function can be passed to the audio_output_callback field of the RealtimeClientBase.""" diff --git a/python/semantic_kernel/contents/binary_content.py b/python/semantic_kernel/contents/binary_content.py index 007186a9363e..c580830601f1 100644 --- a/python/semantic_kernel/contents/binary_content.py +++ b/python/semantic_kernel/contents/binary_content.py @@ -2,6 +2,7 @@ import logging import os +from base64 import b64encode from typing import Annotated, Any, ClassVar, Literal, TypeVar from xml.etree.ElementTree import Element # nosec @@ -185,3 +186,13 @@ def write_to_file(self, path: str | FilePath) -> None: def to_dict(self) -> dict[str, Any]: """Convert the instance to a dictionary.""" return {"type": "binary", "binary": {"uri": str(self)}} + + def to_base64_bytestring(self, encoding: str = "utf-8") -> str: + """Convert the instance to a bytestring.""" + if self._data_uri and self._data_uri.data_array is not None: + return b64encode(self._data_uri.data_array.tobytes()).decode(encoding) + if self._data_uri and self._data_uri.data_bytes: + return self._data_uri.data_bytes.decode(encoding) + if self._data_uri and self._data_uri.data_str: + return self._data_uri.data_str + return "" diff --git a/python/semantic_kernel/contents/events/realtime_event.py b/python/semantic_kernel/contents/events/realtime_event.py index 7de87f078ff6..edb2c5917778 100644 --- a/python/semantic_kernel/contents/events/realtime_event.py +++ b/python/semantic_kernel/contents/events/realtime_event.py @@ -19,7 +19,7 @@ class ServiceEvent(KernelBaseModel): """Base class for all service events.""" - event_type: Literal["service"] + event_type: Literal["service"] = "service" service_type: str event: Any | None = None @@ -27,7 +27,7 @@ class ServiceEvent(KernelBaseModel): class AudioEvent(KernelBaseModel): """Audio event type.""" - event_type: Literal["audio"] + event_type: Literal["audio"] = "audio" service_type: str | None = None audio: AudioContent @@ -35,7 +35,7 @@ class AudioEvent(KernelBaseModel): class TextEvent(KernelBaseModel): """Text event type.""" - event_type: Literal["text"] + event_type: Literal["text"] = "text" service_type: str | None = None text: TextContent @@ -43,7 +43,7 @@ class TextEvent(KernelBaseModel): class FunctionCallEvent(KernelBaseModel): """Function call event type.""" - event_type: Literal["function_call"] + event_type: Literal["function_call"] = "function_call" service_type: str | None = None function_call: FunctionCallContent @@ -51,6 +51,6 @@ class FunctionCallEvent(KernelBaseModel): class FunctionResultEvent(KernelBaseModel): """Function result event type.""" - event_type: Literal["function_result"] + event_type: Literal["function_result"] = "function_result" service_type: str | None = None function_result: FunctionResultContent From ad2ec587a9f19da8cdf7e17385cde5f80f8ec62d Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 31 Jan 2025 15:49:52 +0100 Subject: [PATCH 24/25] removed built-in audio players, split for websocket and rtc --- docs/decisions/00XX-realtime-api-clients.md | 346 ------------- python/samples/concepts/audio/utils.py | 11 - .../01-chat_with_realtime_webrtc.py} | 18 +- .../01-chat_with_realtime_websocket.py | 95 ++++ .../02-chat_with_function_calling.py} | 15 +- python/samples/concepts/realtime/utils.py | 470 ++++++++++++++++++ .../ai/open_ai/services/open_ai_realtime.py | 41 +- .../realtime/open_ai_realtime_base.py | 103 ++-- .../realtime/open_ai_realtime_webrtc.py | 11 +- .../realtime/open_ai_realtime_websocket.py | 10 +- .../ai/open_ai/services/realtime/utils.py | 2 +- .../connectors/ai/realtime_client_base.py | 5 +- .../connectors/ai/utils/__init__.py | 5 - .../connectors/ai/utils/realtime_helpers.py | 267 ---------- .../contents/events/realtime_event.py | 33 +- 15 files changed, 689 insertions(+), 743 deletions(-) delete mode 100644 docs/decisions/00XX-realtime-api-clients.md delete mode 100644 python/samples/concepts/audio/utils.py rename python/samples/concepts/{audio/04-chat_with_realtime_api_simple.py => realtime/01-chat_with_realtime_webrtc.py} (86%) create mode 100644 python/samples/concepts/realtime/01-chat_with_realtime_websocket.py rename python/samples/concepts/{audio/05-chat_with_realtime_api_complex.py => realtime/02-chat_with_function_calling.py} (92%) create mode 100644 python/samples/concepts/realtime/utils.py delete mode 100644 python/semantic_kernel/connectors/ai/utils/__init__.py delete mode 100644 python/semantic_kernel/connectors/ai/utils/realtime_helpers.py diff --git a/docs/decisions/00XX-realtime-api-clients.md b/docs/decisions/00XX-realtime-api-clients.md deleted file mode 100644 index bde864d79b52..000000000000 --- a/docs/decisions/00XX-realtime-api-clients.md +++ /dev/null @@ -1,346 +0,0 @@ ---- -# These are optional elements. Feel free to remove any of them. -status: {proposed } -contact: {Eduard van Valkenburg} -date: {2025-01-10} -deciders: { Eduard van Valkenburg, Mark Wallace, Ben Thomas, Roger Barreto} -consulted: -informed: ---- - -# Realtime API Clients - -## Context and Problem Statement - -Multiple model providers are starting to enable realtime voice-to-voice or even multi-model-to-voice communication with their models, this includes OpenAI with their [Realtime API](https://openai.com/index/introducing-the-realtime-api/) and [Google Gemini](https://ai.google.dev/api/multimodal-live). These API's promise some very interesting new ways of using LLM's in different settings, which we want to enable with Semantic Kernel. - The key feature that Semantic Kernel brings into this system is the ability to (re)use Semantic Kernel function as tools with these API's. There are also options for Google to use video and images as input, but for now we are focusing on the voice-to-voice part, while keeping in mind that video is coming. - -The protocols that these API's use at this time are Websockets and WebRTC. - -In both cases there are events being sent to and from the service, some events contain content, text, audio, or video (so far only sending, not receiving), while some events are "control" events, like content created, function call requested, etc. Sending events include, sending content, either voice, text or function call output, or events, like committing the input audio and requesting a response. - -### Websocket -Websocket has been around for a while and is a well known technology, it is a full-duplex communication protocol over a single, long-lived connection. It is used for sending and receiving messages between client and server in real-time. Each event can contain a message, which might contain a content item, or a control event. Audio is sent as a base64 encoded string. - -### WebRTC -WebRTC is a Mozilla project that provides web browsers and mobile applications with real-time communication via simple application programming interfaces (APIs). It allows audio and video communication to work inside web pages and other applications by allowing direct peer-to-peer communication, eliminating the need to install plugins or download native apps. It is used for sending and receiving audio and video streams, and can be used for sending (data-)messages as well. The big difference compared to websockets is that it explicitly create a channel for audio and video, and a separate channel for "data", which are events but in this space also things like Function calls. - -Both the OpenAI and Google realtime api's are in preview/beta, this means there might be breaking changes in the way they work coming in the future, therefore the clients built to support these API's are going to be experimental until the API's stabilize. - -One feature that we need to consider if and how to deal with is whether or not a service uses Voice Activated Detection, OpenAI supports turning that off and allows parameters for how it behaves, while Google has it on by default and it cannot be configured. - -### Event types (Websocket and partially WebRTC) - -#### Client side events: -| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | -| ------------------------- | --------------------------------- | ---------------------------- | ---------------------------------- | -| Control | Configure session | `session.update` | `BidiGenerateContentSetup` | -| Content | Send voice input | `input_audio_buffer.append` | `BidiGenerateContentRealtimeInput` | -| Control | Commit input and request response | `input_audio_buffer.commit` | `-` | -| Control | Clean audio input buffer | `input_audio_buffer.clear` | `-` | -| Content | Send text input | `conversation.item.create` | `BidiGenerateContentClientContent` | -| Control | Interrupt audio | `conversation.item.truncate` | `-` | -| Control | Delete content | `conversation.item.delete` | `-` | -| Control | Respond to function call request | `conversation.item.create` | `BidiGenerateContentToolResponse` | -| Control | Ask for response | `response.create` | `-` | -| Control | Cancel response | `response.cancel` | `-` | - -#### Server side events: -| **Content/Control event** | **Event Description** | **OpenAI Event** | **Google Event** | -| ------------------------- | -------------------------------------- | ------------------------------------------------------- | ----------------------------------------- | -| Control | Error | `error` | `-` | -| Control | Session created | `session.created` | `BidiGenerateContentSetupComplete` | -| Control | Session updated | `session.updated` | `BidiGenerateContentSetupComplete` | -| Control | Conversation created | `conversation.created` | `-` | -| Control | Input audio buffer committed | `input_audio_buffer.committed` | `-` | -| Control | Input audio buffer cleared | `input_audio_buffer.cleared` | `-` | -| Control | Input audio buffer speech started | `input_audio_buffer.speech_started` | `-` | -| Control | Input audio buffer speech stopped | `input_audio_buffer.speech_stopped` | `-` | -| Content | Conversation item created | `conversation.item.created` | `-` | -| Content | Input audio transcription completed | `conversation.item.input_audio_transcription.completed` | | -| Content | Input audio transcription failed | `conversation.item.input_audio_transcription.failed` | | -| Control | Conversation item truncated | `conversation.item.truncated` | `-` | -| Control | Conversation item deleted | `conversation.item.deleted` | `-` | -| Control | Response created | `response.created` | `-` | -| Control | Response done | `response.done` | `-` | -| Content | Response output item added | `response.output_item.added` | `-` | -| Content | Response output item done | `response.output_item.done` | `-` | -| Content | Response content part added | `response.content_part.added` | `-` | -| Content | Response content part done | `response.content_part.done` | `-` | -| Content | Response text delta | `response.text.delta` | `BidiGenerateContentServerContent` | -| Content | Response text done | `response.text.done` | `-` | -| Content | Response audio transcript delta | `response.audio_transcript.delta` | `BidiGenerateContentServerContent` | -| Content | Response audio transcript done | `response.audio_transcript.done` | `-` | -| Content | Response audio delta | `response.audio.delta` | `BidiGenerateContentServerContent` | -| Content | Response audio done | `response.audio.done` | `-` | -| Content | Response function call arguments delta | `response.function_call_arguments.delta` | `BidiGenerateContentToolCall` | -| Content | Response function call arguments done | `response.function_call_arguments.done` | `-` | -| Control | Function call cancelled | `-` | `BidiGenerateContentToolCallCancellation` | -| Control | Rate limits updated | `rate_limits.updated` | `-` | - - -## Overall Decision Drivers -- Simple programming model that is likely able to handle future realtime api's and the evolution of the existing ones. -- Whenever possible we transform incoming content into Semantic Kernel content, but surface everything, so it's extensible -- Protocol agnostic, should be able to use different types of protocols under the covers, like websocket and WebRTC, without changing the client code (unless the protocol requires it), there will be slight differences in behavior depending on the protocol. - -There are multiple areas where we need to make decisions, these are: -- Content and Events -- Programming model -- Audio speaker/microphone handling - -# Content and Events - -## Considered Options - Content and Events -Both the sending and receiving side of these integrations need to decide how to deal with the events. - -1. Treat content events separate from control events -1. Treat everything as content items -1. Treat everything as events - -### 1. Treat content events separate from control events -This would mean there are two mechanisms in the clients, one deals with content, and one with control events. - -- Pro: - - strongly typed responses for known content - - easy to use as the main interactions are clear with familiar SK content types, the rest goes through a separate mechanism -- Con: - - new content support requires updates in the codebase and can be considered breaking (potentially sending additional types back) - - additional complexity in dealing with two streams of data - -### 2. Treat everything as content items -This would mean that all events are turned into Semantic Kernel content items, and would also mean that we need to define additional content types for the control events. - -- Pro: - - everything is a content item, so it's easy to deal with -- Con: - - overkill for simple control events - -### 3. Treat everything as events -This would mean that all events are retained and returned to the developer as is, without any transformation. - -- Pro: - - no transformation needed - - easy to maintain -- Con: - - nothing easing the burden on the developer, they need to deal with the raw events - - no way to easily switch between one provider and another - -## Decision Outcome - Content and Events - -Chosen option: 3 Treat Everything as Events - -This option was chosen to allow abstraction away from the raw events, while still allowing the developer to access the raw events if needed. -A set of events are defined, for basic types, like 'audio', 'text', 'function_call', 'function_result', it then has two other fields, service_event which is filled with the event type from the service and a field for the actual content, with a name that corresponds to the event type: - -```python -AudioEvent( - event_type="audio", - service_event= "response.audio.delta", - audio: AudioContent(...) -) -``` - -Next to these we will have a generic event, called ServiceEvent, this is the catch-all, which has event_type: "service", the service_event field filled with the event type from the service and a field called 'event' which contains the raw event from the service. - -```python -ServiceEvent( - event_type="service", - service_event= "conversation.item.create", - event: { ... } -) -``` - -This allows you to easily filter on the event_type, and then use the service_event to filter on the specific event type, and then use the content field to get the content, or the event field to get the raw event. - -Collectively these are known as *RealtimeEvents*, and are returned as an async generator from the client, so you can easily loop over them. And they are passed to the send method. - -Initially RealtimeEvents are: -- AudioEvent -- TextEvent -- FunctionCallEvent -- FunctionResultEvent -- ServiceEvent - -# Programming model - -## Considered Options - Programming model -The programming model for the clients needs to be simple and easy to use, while also being able to handle the complexity of the realtime api's. - -_In this section we will refer to events for both content and events, regardless of the decision made in the previous section._ - -1. Async generator for receiving events, that yields Events, combined with a event handler/callback mechanism for receiving events and a function for sending events - - 1a: Single event handlers, where each event is passed to the handler - - 1b: Multiple event handlers, where each event type has its own handler -2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers -3. Purely a start listening method that yields Events, and a send method that sends events - -### 1. Async generator for receiving events, that yields contents, combined with a event handler/callback mechanism for receiving events and a function for sending events -This would mean that the client would have a mechanism to register event handlers, and the integration would call these handlers when an event is received. For sending events, a function would be created that sends the event to the service. - -- Pro: - - without any additional setup you get content back, just as with "regular" chat models - - event handlers are mostly for more complex interactions, so ok to be slightly more complex -- Con: - - developer judgement needs to be made (or exposed with parameters) on what is returned through the async generator and what is passed to the event handlers - -### 2. Event buffers/queues that are exposed to the developer, start sending and start receiving methods, that just initiate the sending and receiving of events and thereby the filling of the buffers -This would mean that there are two queues, one for sending and one for receiving, and the developer can listen to the receiving queue and send to the sending queue. Internal things like parsing events to content types and auto-function calling are processed first, and the result is put in the queue, the content type should use inner_content to capture the full event and these might add a message to the send queue as well. - -- Pro: - - simple to use, just start sending and start receiving - - easy to understand, as queues are a well known concept - - developers can just skip events they are not interested in -- Con: - - potentially causes audio delays because of the queueing mechanism - -### 2b. Same as option 2, but with priority handling of audio content -This would mean that the audio content is handled, and passed to the developer code, and then all other events are processed. - -- Pro: - - mitigates audio delays - - easy to understand, as queues are a well known concept - - developers can just skip events they are not interested in -- Con: - - Two separate mechanisms used for audio content and events - -## Decision Outcome - Programming model - -Chosen option: Purely a start listening method that yields Events, and a send method that sends events - -This makes the programming model very easy, a minimal setup that should work for every service and protocol would look like this: -```python -async for event in realtime_client.start_streaming(): - match event.event_type: - case "audio": - await audio_player.add_audio(event.audio) - case "text": - print(event.text.text) -``` - - -# Audio speaker/microphone handling - -## Considered Options - Audio speaker/microphone handling - -1. Create abstraction in SK for audio handlers, that can be passed into the realtime client to record and play audio -2. Send and receive AudioContent (potentially wrapped in StreamingChatMessageContent) to the client, and let the client handle the audio recording and playing - -### 1. Create abstraction in SK for audio handlers, that can be passed into the realtime client to record and play audio -This would mean that the client would have a mechanism to register audio handlers, and the integration would call these handlers when audio is received or needs to be sent. A additional abstraction for this would have to be created in Semantic Kernel (or potentially taken from a standard). - -- Pro: - - simple/local audio handlers can be shipped with SK making it easy to use - - extensible by third parties to integrate into other systems (like Azure Communications Service) - - could mitigate buffer issues by prioritizing audio content being sent to the handlers -- Con: - - extra code in SK that needs to be maintained, potentially relying on third party code - -### 2. Send and receive AudioContent (wrapped in StreamingChatMessageContent) to the client, and let the client handle the audio recording and playing -This would mean that the client would receive AudioContent items, and would have to deal with them itself, including recording and playing the audio. - -- Pro: - - no extra code in SK that needs to be maintained -- Con: - - extra burden on the developer to deal with the audio - - harder to get started with - -## Decision Outcome - Audio speaker/microphone handling - -Chosen option: ... - -# Interface design - -## Considered Options - Interface design - -1. Use a single class for everything -2. Split the service class from a session class. - -The following methods will need to be supported: -- create session -- update session -- close session -- listen for/receive events -- send events - -### 1. Use a single class for everything - -Each implementation would have to implements all of the above methods. This means that non-protocol specific elements are in the same class as the protocol specific elements and will lead to code duplication between them. - -### 2. Split the service class from a session class. - -Two interfaces are created: -- Service: create session, update session, delete session, list sessions -- Session: listen for/receive events, send events, update session, close session - -Currently neither the google or the openai api's support restarting sessions, so the advantage of splitting is mostly a implementation question but will not add any benefits to the user. - -This means that the split would be far simpler: -- Service: create session -- Session: listen for/receive events, send events, update session, close session - -## Naming - -The send and listen/receive methods need to be clear in the way their are named and this can become confusing when dealing with these api's. The following options are considered: - -Options for sending events to the service from your code: -- google uses .send in their client. -- OpenAI uses .send in their client as well -- send or send_message is used in other clients, like Azure Communication Services - -Options for listening for events from the service in your code: -- google uses .receive in their client. -- openai uses .recv in their client. -- others use receive or receive_messages in their clients. - -### Decision Outcome - Interface design - -Chosen option: Use a single class for everything -Chosen for send and receive as verbs. - -This means that the interface will look like this: -```python - -class RealtimeClient: - async def create_session(self, settings: PromptExecutionSettings, chat_history: ChatHistory, **kwargs) -> None: - ... - - async def update_session(self, settings: PromptExecutionSettings, chat_history: ChatHistory, **kwargs) -> None: - ... - - async def close_session(self, **kwargs) -> None: - ... - - async def receive(self, **kwargs) -> AsyncGenerator[RealtimeEvent, None]: - ... - - async def send(self, event: RealtimeEvent) -> None: - ... -``` - -In most cases, create_session should call update_session with the same parameters, since update session can also be done separately later on with the same inputs. - -For Python a default __aenter__ and __aexit__ method should be added to the class, so it can be used in a with statement, which calls create_session and close_session respectively. - -It is advisable, but not required, to implement the send method through a buffer/queue so that events be can 'sent' before the sessions has been established without losing them or raising exceptions, this might take a very seconds and in that time a single send call would block the application. - -For receiving a internal implementation might also rely on a buffer/queue, but this is up to the developer and what makes sense for that service. For instance webrtc relies on defining the callback at create session time, so the create_session method adds a function that adds events to the queue and the receive method starts reading from and yielding from that queue. - -The send method should handle all events types, but it might have to handle the same thing in two ways, for instance: -```python -audio = AudioContent(...) - -await client.send(AudioEvent(event_type='audio', audio=audio)) -``` - -is equivalent to (at least in the case of OpenAI): -```python -audio = AudioContent(...) - -await client.send(ServiceEvent(event_type='service', service_event='input_audio_buffer.append', event=audio)) -``` - -The first version allows one to have the exact same code for all services, while the second version is also correct and should be handled correctly as well, this once again allows for flexibility and simplicity, when audio needs to be sent to with a different event type, that is still possible in the second way, while the first uses the "default" event type for that particular service. - - - diff --git a/python/samples/concepts/audio/utils.py b/python/samples/concepts/audio/utils.py deleted file mode 100644 index fda9ecb7d772..000000000000 --- a/python/samples/concepts/audio/utils.py +++ /dev/null @@ -1,11 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -import logging - -import sounddevice as sd - -logger = logging.getLogger(__name__) - - -def check_audio_devices(): - logger.debug(sd.query_devices()) diff --git a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py b/python/samples/concepts/realtime/01-chat_with_realtime_webrtc.py similarity index 86% rename from python/samples/concepts/audio/04-chat_with_realtime_api_simple.py rename to python/samples/concepts/realtime/01-chat_with_realtime_webrtc.py index 06ee11807a81..38d3803a737b 100644 --- a/python/samples/concepts/audio/04-chat_with_realtime_api_simple.py +++ b/python/samples/concepts/realtime/01-chat_with_realtime_webrtc.py @@ -3,16 +3,17 @@ import asyncio import logging -from samples.concepts.audio.utils import check_audio_devices +from samples.concepts.realtime.utils import AudioPlayerWebRTC, AudioRecorderWebRTC, check_audio_devices from semantic_kernel.connectors.ai.open_ai import ( ListenEvents, OpenAIRealtime, OpenAIRealtimeExecutionSettings, TurnDetection, ) -from semantic_kernel.connectors.ai.utils import SKAudioPlayer logging.basicConfig(level=logging.WARNING) +utils_log = logging.getLogger("samples.concepts.realtime.utils") +utils_log.setLevel(logging.INFO) aiortc_log = logging.getLogger("aiortc") aiortc_log.setLevel(logging.WARNING) aioice_log = logging.getLogger("aioice") @@ -43,7 +44,12 @@ async def main() -> None: # create the realtime client and optionally add the audio output function, this is optional # you can define the protocol to use, either "websocket" or "webrtc" # they will behave the same way, even though the underlying protocol is quite different - realtime_client = OpenAIRealtime("webrtc") + audio_player = AudioPlayerWebRTC() + realtime_client = OpenAIRealtime( + "webrtc", + audio_output_callback=audio_player.client_callback, + audio_track=AudioRecorderWebRTC(), + ) # Create the settings for the session settings = OpenAIRealtimeExecutionSettings( instructions=""" @@ -58,15 +64,15 @@ async def main() -> None: turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), ) # the context manager calls the create_session method on the client and start listening to the audio stream - audio_player = SKAudioPlayer() + print("Mosscap (transcript): ", end="") async with realtime_client, audio_player: await realtime_client.update_session(settings=settings, create_response=True) async for event in realtime_client.receive(): match event.event_type: - case "audio": - await audio_player.add_audio(event.audio) + # case "audio": + # await audio_player.add_audio(event.audio) case "text": print(event.text.text, end="") case "service": diff --git a/python/samples/concepts/realtime/01-chat_with_realtime_websocket.py b/python/samples/concepts/realtime/01-chat_with_realtime_websocket.py new file mode 100644 index 000000000000..e647da6ff4a9 --- /dev/null +++ b/python/samples/concepts/realtime/01-chat_with_realtime_websocket.py @@ -0,0 +1,95 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import logging + +from samples.concepts.realtime.utils import AudioPlayerWebsocket, AudioRecorderWebsocket, check_audio_devices +from semantic_kernel.connectors.ai.open_ai import ( + ListenEvents, + OpenAIRealtime, + OpenAIRealtimeExecutionSettings, + TurnDetection, +) + +logging.basicConfig(level=logging.WARNING) +utils_log = logging.getLogger("samples.concepts.realtime.utils") +utils_log.setLevel(logging.INFO) +aiortc_log = logging.getLogger("aiortc") +aiortc_log.setLevel(logging.WARNING) +aioice_log = logging.getLogger("aioice") +aioice_log.setLevel(logging.WARNING) +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + +# This simple sample demonstrates how to use the OpenAI Realtime API to create +# a chat bot that can listen and respond directly through audio. +# It requires installing: +# - semantic-kernel[openai_realtime] +# - pyaudio +# - sounddevice +# - pydub +# - aiortc +# e.g. pip install pyaudio sounddevice pydub + +# The characterics of your speaker and microphone are a big factor in a smooth conversation +# so you may need to try out different devices for each. +# you can also play around with the turn_detection settings to get the best results. +# It has device id's set in the AudioRecorderStream and AudioPlayerAsync classes, +# so you may need to adjust these for your system. +# you can check the available devices by uncommenting line below the function +check_audio_devices() + + +async def main() -> None: + # create the realtime client and optionally add the audio output function, this is optional + # you can define the protocol to use, either "websocket" or "webrtc" + # they will behave the same way, even though the underlying protocol is quite different + audio_player = AudioPlayerWebsocket() + realtime_client = OpenAIRealtime( + "websocket", + audio_output_callback=audio_player.client_callback, + ) + audio_recorder = AudioRecorderWebsocket(realtime_client=realtime_client) + # Create the settings for the session + settings = OpenAIRealtimeExecutionSettings( + instructions=""" + You are a chat bot. Your name is Mosscap and + you have one goal: figure out what people need. + Your full name, should you need to know it, is + Splendid Speckled Mosscap. You communicate + effectively, but you tend to answer with long + flowery prose. + """, + voice="shimmer", + turn_detection=TurnDetection(type="server_vad", create_response=True, silence_duration_ms=800, threshold=0.8), + ) + # the context manager calls the create_session method on the client and start listening to the audio stream + print("Mosscap (transcript): ", end="") + + async with realtime_client, audio_player, audio_recorder: + await realtime_client.update_session(settings=settings, create_response=True) + + async for event in realtime_client.receive(): + match event.event_type: + # this can be used as an alternative to the callback function used above, + # the callback is faster and smoother + # case "audio": + # await audio_player.add_audio(event.audio) + case "text": + print(event.text.text, end="") + case "service": + # OpenAI Specific events + if event.service_type == ListenEvents.SESSION_UPDATED: + print("Session updated") + if event.service_type == ListenEvents.RESPONSE_CREATED: + print("") + if event.service_type == ListenEvents.ERROR: + logger.error(event.event) + + +if __name__ == "__main__": + print( + "Instruction: start speaking, when you stop the API should detect you finished and start responding. " + "Press ctrl + c to stop the program." + ) + asyncio.run(main()) diff --git a/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py b/python/samples/concepts/realtime/02-chat_with_function_calling.py similarity index 92% rename from python/samples/concepts/audio/05-chat_with_realtime_api_complex.py rename to python/samples/concepts/realtime/02-chat_with_function_calling.py index b567c0028178..c74b6b583d23 100644 --- a/python/samples/concepts/audio/05-chat_with_realtime_api_complex.py +++ b/python/samples/concepts/realtime/02-chat_with_function_calling.py @@ -5,7 +5,7 @@ from datetime import datetime from random import randint -from samples.concepts.audio.utils import check_audio_devices +from samples.concepts.realtime.utils import AudioPlayerWebRTC, AudioRecorderWebRTC, check_audio_devices from semantic_kernel import Kernel from semantic_kernel.connectors.ai import FunctionChoiceBehavior from semantic_kernel.connectors.ai.open_ai import ( @@ -14,11 +14,12 @@ OpenAIRealtimeExecutionSettings, TurnDetection, ) -from semantic_kernel.connectors.ai.utils import SKAudioPlayer, SKAudioTrack from semantic_kernel.contents import ChatHistory from semantic_kernel.functions import kernel_function logging.basicConfig(level=logging.WARNING) +utils_log = logging.getLogger("samples.concepts.realtime.utils") +utils_log.setLevel(logging.INFO) aiortc_log = logging.getLogger("aiortc") aiortc_log.setLevel(logging.WARNING) aioice_log = logging.getLogger("aioice") @@ -78,15 +79,15 @@ async def main() -> None: # create the audio player and audio track # both take a device_id parameter, which is the index of the device to use, if None the default device is used - audio_player = SKAudioPlayer(sample_rate=24000, frame_duration=100, channels=1) - audio_track = SKAudioTrack() + audio_player = AudioPlayerWebRTC() + audio_track = AudioRecorderWebRTC() # create the realtime client and optionally add the audio output function, this is optional # you can define the protocol to use, either "websocket" or "webrtc" # they will behave the same way, even though the underlying protocol is quite different realtime_client = OpenAIRealtime( - protocol="websocket", + protocol="webrtc", audio_output_callback=audio_player.client_callback, - # audio_track=audio_track, + audio_track=audio_track, ) # Create the settings for the session @@ -116,7 +117,7 @@ async def main() -> None: chat_history.add_assistant_message("I am Mosscap, a chat bot. I'm trying to figure out what people need.") # the context manager calls the create_session method on the client and start listening to the audio stream - async with realtime_client, audio_player, audio_track.stream_to_realtime_client(realtime_client): + async with realtime_client, audio_player: await realtime_client.update_session( settings=settings, chat_history=chat_history, kernel=kernel, create_response=True ) diff --git a/python/samples/concepts/realtime/utils.py b/python/samples/concepts/realtime/utils.py new file mode 100644 index 000000000000..d7f39369a0d4 --- /dev/null +++ b/python/samples/concepts/realtime/utils.py @@ -0,0 +1,470 @@ +# Copyright (c) Microsoft. All rights reserved. + +import asyncio +import base64 +import logging +import threading +from typing import Any, ClassVar, Final, cast + +import numpy as np +import numpy.typing as npt +import sounddevice as sd +from aiortc.mediastreams import MediaStreamError, MediaStreamTrack +from av.audio.frame import AudioFrame +from av.frame import Frame +from pydantic import PrivateAttr +from sounddevice import InputStream, OutputStream + +from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase +from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.events.realtime_event import AudioEvent +from semantic_kernel.kernel_pydantic import KernelBaseModel + +logger = logging.getLogger(__name__) + +SAMPLE_RATE: Final[int] = 24000 +RECORDER_CHANNELS: Final[int] = 1 +PLAYER_CHANNELS: Final[int] = 1 +FRAME_DURATION: Final[int] = 100 +SAMPLE_RATE_WEBRTC: Final[int] = 48000 +RECORDER_CHANNELS_WEBRTC: Final[int] = 1 +PLAYER_CHANNELS_WEBRTC: Final[int] = 2 +FRAME_DURATION_WEBRTC: Final[int] = 20 +DTYPE: Final[npt.DTypeLike] = np.int16 + + +def check_audio_devices(): + logger.info(sd.query_devices()) + + +# region: Recorders + + +class AudioRecorderWebRTC(KernelBaseModel, MediaStreamTrack): + """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice.""" + + kind: ClassVar[str] = "audio" + device: str | int | None = None + sample_rate: int + channels: int + frame_duration: int + dtype: npt.DTypeLike = DTYPE + frame_size: int = 0 + _queue: asyncio.Queue[Frame] = PrivateAttr(default_factory=asyncio.Queue) + _is_recording: bool = False + _stream: InputStream | None = None + _recording_task: asyncio.Task | None = None + _loop: asyncio.AbstractEventLoop | None = None + _pts: int = 0 + + def __init__( + self, + *, + device: str | int | None = None, + sample_rate: int = SAMPLE_RATE_WEBRTC, + channels: int = RECORDER_CHANNELS_WEBRTC, + frame_duration: int = FRAME_DURATION_WEBRTC, + dtype: npt.DTypeLike = DTYPE, + ): + """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. + + Make sure the device is set to the correct device for your system. + + Args: + device: The device id to use for recording audio. + sample_rate: The sample rate for the audio. + channels: The number of channels for the audio. + frame_duration: The duration of each audio frame in milliseconds. + dtype: The data type for the audio. + """ + super().__init__(**{ + "device": device, + "sample_rate": sample_rate, + "channels": channels, + "frame_duration": frame_duration, + "dtype": dtype, + "frame_size": int(sample_rate * frame_duration / 1000), + }) + MediaStreamTrack.__init__(self) + + async def recv(self) -> Frame: + """Receive the next frame of audio data.""" + if not self._recording_task: + self._recording_task = asyncio.create_task(self.start_recording()) + + try: + frame = await self._queue.get() + self._queue.task_done() + return frame + except Exception as e: + logger.error(f"Error receiving audio frame: {e!s}") + raise MediaStreamError("Failed to receive audio frame") + + def _sounddevice_callback(self, indata: np.ndarray, frames: int, time: Any, status: Any) -> None: + if status: + logger.warning(f"Audio input status: {status}") + if self._loop and self._loop.is_running(): + asyncio.run_coroutine_threadsafe(self._queue.put(self._create_frame(indata)), self._loop) + + def _create_frame(self, indata: np.ndarray) -> Frame: + audio_data = indata.copy() + if audio_data.dtype != self.dtype: + audio_data = ( + (audio_data * 32767).astype(self.dtype) if self.dtype == np.int16 else audio_data.astype(self.dtype) + ) + frame = AudioFrame( + format="s16", + layout="mono", + samples=len(audio_data), + ) + frame.rate = self.sample_rate + frame.pts = self._pts + frame.planes[0].update(audio_data.tobytes()) + self._pts += len(audio_data) + return frame + + async def start_recording(self): + """Start recording audio from the input device.""" + if self._is_recording: + return + + self._is_recording = True + self._loop = asyncio.get_running_loop() + self._pts = 0 # Reset pts when starting recording + + try: + self._stream = InputStream( + device=self.device, + channels=self.channels, + samplerate=self.sample_rate, + dtype=self.dtype, + blocksize=self.frame_size, + callback=self._sounddevice_callback, + ) + self._stream.start() + + while self._is_recording: + await asyncio.sleep(0.1) + except asyncio.CancelledError | KeyboardInterrupt: + logger.debug("Recording task was stopped.") + except Exception as e: + logger.error(f"Error in audio recording: {e!s}") + raise + finally: + self._is_recording = False + + +class AudioRecorderWebsocket(KernelBaseModel): + """A simple class that implements a sounddevice for use with websockets.""" + + realtime_client: RealtimeClientBase + device: str | int | None = None + sample_rate: int + channels: int + frame_duration: int + dtype: npt.DTypeLike = DTYPE + frame_size: int = 0 + _stream: InputStream | None = None + _pts: int = 0 + _stream_task: asyncio.Task | None = None + + def __init__( + self, + *, + realtime_client: RealtimeClientBase, + device: str | int | None = None, + sample_rate: int = SAMPLE_RATE, + channels: int = RECORDER_CHANNELS, + frame_duration: int = FRAME_DURATION, + dtype: npt.DTypeLike = DTYPE, + ): + """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. + + Make sure the device is set to the correct device for your system. + + Args: + realtime_client: The RealtimeClientBase to use for streaming audio. + device: The device id to use for recording audio. + sample_rate: The sample rate for the audio. + channels: The number of channels for the audio. + frame_duration: The duration of each audio frame in milliseconds. + dtype: The data type for the audio. + **kwargs: Additional keyword arguments. + """ + super().__init__(**{ + "realtime_client": realtime_client, + "device": device, + "sample_rate": sample_rate, + "channels": channels, + "frame_duration": frame_duration, + "dtype": dtype, + "frame_size": int(sample_rate * frame_duration / 1000), + }) + + async def __aenter__(self): + """Stream audio data to a RealtimeClientBase.""" + if not self._stream_task: + self._stream_task = asyncio.create_task(self._start_stream()) + return self + + async def _start_stream(self): + self._pts = 0 # Reset pts when starting recording + self._stream = InputStream( + device=self.device, + channels=self.channels, + samplerate=self.sample_rate, + dtype=self.dtype, + blocksize=self.frame_size, + ) + self._stream.start() + try: + while True: + if self._stream.read_available < self.frame_size: + await asyncio.sleep(0) + continue + data, _ = self._stream.read(self.frame_size) + + await self.realtime_client.send( + AudioEvent(audio=AudioContent(data=base64.b64encode(cast(Any, data)).decode("utf-8"))) + ) + + await asyncio.sleep(0) + except asyncio.CancelledError: + pass + + async def __aexit__(self, exc_type, exc, tb): + """Stop recording audio.""" + if self._stream_task: + self._stream_task.cancel() + await self._stream_task + if self._stream: + self._stream.stop() + self._stream.close() + + +# region: Players + + +class AudioPlayerWebRTC(KernelBaseModel): + """Simple class that plays audio using sounddevice. + + Make sure the device_id is set to the correct device for your system. + + The sample rate, channels and frame duration + should be set to match the audio you + are receiving. + + Args: + device: The device id to use for playing audio. + sample_rate: The sample rate for the audio. + channels: The number of channels for the audio. + dtype: The data type for the audio. + frame_duration: The duration of each audio frame in milliseconds + + """ + + device: int | None = None + sample_rate: int = SAMPLE_RATE_WEBRTC + channels: int = PLAYER_CHANNELS_WEBRTC + dtype: npt.DTypeLike = DTYPE + frame_duration: int = FRAME_DURATION_WEBRTC + _queue: asyncio.Queue[np.ndarray] | None = PrivateAttr(default=None) + _stream: OutputStream | None = PrivateAttr(default=None) + + async def __aenter__(self): + """Start the audio stream when entering a context.""" + self.start() + return self + + async def __aexit__(self, exc_type, exc, tb): + """Stop the audio stream when exiting a context.""" + self.stop() + + def start(self): + """Start the audio stream.""" + self._queue = asyncio.Queue() + self._stream = OutputStream( + callback=self._sounddevice_callback, + samplerate=self.sample_rate, + channels=self.channels, + dtype=self.dtype, + blocksize=int(self.sample_rate * self.frame_duration / 1000), + device=self.device, + ) + if self._stream and self._queue: + self._stream.start() + + def stop(self): + """Stop the audio stream.""" + if self._stream: + self._stream.stop() + self._stream = None + self._queue = None + + def _sounddevice_callback(self, outdata, frames, time, status): + """This callback is called by sounddevice when it needs more audio data to play.""" + if status: + logger.debug(f"Audio output status: {status}") + if self._queue: + if self._queue.empty(): + return + data = self._queue.get_nowait() + outdata[:] = data.reshape(outdata.shape) + self._queue.task_done() + else: + logger.error( + "Audio queue not initialized, make sure to call start before " + "using the player, or use the context manager." + ) + + async def client_callback(self, content: np.ndarray): + """This function can be passed to the audio_output_callback field of the RealtimeClientBase.""" + if self._queue: + await self._queue.put(content) + else: + logger.error( + "Audio queue not initialized, make sure to call start before " + "using the player, or use the context manager." + ) + + async def add_audio(self, audio_content: AudioContent) -> None: + """This function is used to add audio to the queue for playing. + + It first checks if there is a AudioFrame in the inner_content of the AudioContent. + If not, it checks if the data is a numpy array, bytes, or a string and converts it to a numpy array. + """ + if not self._queue: + logger.error( + "Audio queue not initialized, make sure to call start before " + "using the player, or use the context manager." + ) + return + if audio_content.inner_content and isinstance(audio_content.inner_content, AudioFrame): + await self._queue.put(audio_content.inner_content.to_ndarray()) + return + if isinstance(audio_content.data, np.ndarray): + await self._queue.put(audio_content.data) + return + if isinstance(audio_content.data, bytes): + await self._queue.put(np.frombuffer(audio_content.data, dtype=self.dtype)) + return + if isinstance(audio_content.data, str): + await self._queue.put(np.frombuffer(audio_content.data.encode(), dtype=self.dtype)) + return + logger.error(f"Unknown audio content: {audio_content}") + + +class AudioPlayerWebsocket(KernelBaseModel): + """Simple class that plays audio using sounddevice. + + Make sure the device_id is set to the correct device for your system. + + The sample rate, channels and frame duration + should be set to match the audio you + are receiving. + + Args: + device: The device id to use for playing audio. + sample_rate: The sample rate for the audio. + channels: The number of channels for the audio. + dtype: The data type for the audio. + frame_duration: The duration of each audio frame in milliseconds + + """ + + device: int | None = None + sample_rate: int = SAMPLE_RATE + channels: int = PLAYER_CHANNELS + dtype: npt.DTypeLike = DTYPE + frame_duration: int = FRAME_DURATION + _lock: Any = PrivateAttr(default_factory=threading.Lock) + _queue: list[np.ndarray] = PrivateAttr(default_factory=list) + _stream: OutputStream | None = PrivateAttr(default=None) + _frame_count: int = 0 + + async def __aenter__(self): + """Start the audio stream when entering a context.""" + self.start() + return self + + async def __aexit__(self, exc_type, exc, tb): + """Stop the audio stream when exiting a context.""" + self.stop() + + def start(self): + """Start the audio stream.""" + with self._lock: + self._queue = [] + self._stream = OutputStream( + callback=self._sounddevice_callback, + samplerate=self.sample_rate, + channels=self.channels, + dtype=self.dtype, + blocksize=int(self.sample_rate * self.frame_duration / 1000), + device=self.device, + ) + if self._stream: + self._stream.start() + + def stop(self): + """Stop the audio stream.""" + if self._stream: + self._stream.stop() + self._stream = None + with self._lock: + self._queue = [] + + def _sounddevice_callback(self, outdata, frames, time, status): + """This callback is called by sounddevice when it needs more audio data to play.""" + with self._lock: + if status: + logger.debug(f"Audio output status: {status}") + data = np.empty(0, dtype=np.int16) + + # get next item from queue if there is still space in the buffer + while len(data) < frames and len(self._queue) > 0: + item = self._queue.pop(0) + frames_needed = frames - len(data) + data = np.concatenate((data, item[:frames_needed])) + if len(item) > frames_needed: + self._queue.insert(0, item[frames_needed:]) + + self._frame_count += len(data) + + # fill the rest of the frames with zeros if there is no more data + if len(data) < frames: + data = np.concatenate((data, np.zeros(frames - len(data), dtype=np.int16))) + + outdata[:] = data.reshape(-1, 1) + + def reset_frame_count(self): + self._frame_count = 0 + + def get_frame_count(self): + return self._frame_count + + async def client_callback(self, content: np.ndarray): + """This function can be passed to the audio_output_callback field of the RealtimeClientBase.""" + with self._lock: + self._queue.append(content) + + async def add_audio(self, audio_content: AudioContent) -> None: + """This function is used to add audio to the queue for playing. + + It first checks if there is a AudioFrame in the inner_content of the AudioContent. + If not, it checks if the data is a numpy array, bytes, or a string and converts it to a numpy array. + """ + with self._lock: + if audio_content.inner_content and isinstance(audio_content.inner_content, AudioFrame): + self._queue.append(audio_content.inner_content.to_ndarray()) + return + if isinstance(audio_content.data, np.ndarray): + self._queue.append(audio_content.data) + return + if isinstance(audio_content.data, bytes): + self._queue.append(np.frombuffer(audio_content.data, dtype=self.dtype)) + return + if isinstance(audio_content.data, str): + self._queue.append(np.frombuffer(audio_content.data.encode(), dtype=self.dtype)) + return + logger.error(f"Unknown audio content: {audio_content}") diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py index b9e809d4e396..7d6f60eafbd2 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/open_ai_realtime.py @@ -1,6 +1,6 @@ # Copyright (c) Microsoft. All rights reserved. -from collections.abc import AsyncGenerator, Callable, Coroutine, Mapping +from collections.abc import Callable, Coroutine, Mapping from typing import TYPE_CHECKING, Any, ClassVar, Literal, TypeVar from numpy import ndarray @@ -15,17 +15,11 @@ OpenAIRealtimeWebsocketBase, ) from semantic_kernel.connectors.ai.open_ai.settings.open_ai_settings import OpenAISettings -from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings -from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.contents.chat_history import ChatHistory -from semantic_kernel.contents.events.realtime_event import RealtimeEvent from semantic_kernel.exceptions.service_exceptions import ServiceInitializationError if TYPE_CHECKING: from aiortc.mediastreams import MediaStreamTrack - from semantic_kernel.connectors.ai import PromptExecutionSettings - from semantic_kernel.contents import ChatHistory _T = TypeVar("_T", bound="OpenAIRealtime") @@ -33,36 +27,7 @@ __all__ = ["OpenAIRealtime"] -class RealtimeClientStub(RealtimeClientBase): - """This class makes sure that IDE's don't complain about missing methods in the below superclass.""" - - async def send(self, event: Any) -> None: - pass - - async def create_session( - self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, - **kwargs: Any, - ) -> None: - pass - - def receive(self, **kwargs: Any) -> AsyncGenerator[RealtimeEvent, None]: - pass - - async def update_session( - self, - settings: "PromptExecutionSettings | None" = None, - chat_history: "ChatHistory | None" = None, - **kwargs: Any, - ) -> None: - pass - - async def close_session(self) -> None: - pass - - -class OpenAIRealtime(OpenAIRealtimeBase, RealtimeClientStub): +class OpenAIRealtime(OpenAIRealtimeBase): """OpenAI Realtime service.""" def __new__(cls: type["_T"], protocol: str, *args: Any, **kwargs: Any) -> "_T": @@ -128,6 +93,8 @@ def __init__( raise ServiceInitializationError("Failed to create OpenAI settings.", ex) from ex if not openai_settings.realtime_model_id: raise ServiceInitializationError("The OpenAI text model ID is required.") + if audio_track: + kwargs["audio_track"] = audio_track super().__init__( protocol=protocol, audio_output_callback=audio_output_callback, diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py index 0e94dd9c6854..2789bf0d16e2 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_base.py @@ -3,15 +3,18 @@ import json import logging import sys -from collections.abc import AsyncGenerator, Callable, Coroutine +from collections.abc import AsyncGenerator, Callable from typing import TYPE_CHECKING, Any, ClassVar, Literal +from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( + OpenAIRealtimeExecutionSettings, +) + if sys.version_info >= (3, 12): from typing import override # pragma: no cover else: from typing_extensions import override # pragma: no cover -from numpy import ndarray from openai.types.beta.realtime import ( RealtimeClientEvent, RealtimeServerEvent, @@ -27,7 +30,7 @@ from semantic_kernel.connectors.ai.open_ai.services.open_ai_handler import OpenAIHandler from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents, SendEvents from semantic_kernel.connectors.ai.open_ai.services.realtime.utils import ( - _create_realtime_client_event, + _create_openai_realtime_client_event, update_settings_from_function_call_configuration, ) from semantic_kernel.connectors.ai.prompt_execution_settings import PromptExecutionSettings @@ -60,9 +63,8 @@ class OpenAIRealtimeBase(OpenAIHandler, RealtimeClientBase): """OpenAI Realtime service.""" - protocol: ClassVar[Literal["websocket", "webrtc"]] = "websocket" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = True - audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None + protocol: ClassVar[Literal["websocket", "webrtc"]] = "websocket" kernel: Kernel | None = None _current_settings: PromptExecutionSettings | None = PrivateAttr(default=None) @@ -77,7 +79,6 @@ async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[Realt match event.type: case ListenEvents.RESPONSE_AUDIO_TRANSCRIPT_DELTA.value: yield TextEvent( - event_type="text", service_type=event.type, text=StreamingTextContent( inner_content=event, @@ -90,7 +91,6 @@ async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[Realt self._call_id_to_function_map[event.item.call_id] = event.item.name case ListenEvents.RESPONSE_FUNCTION_CALL_ARGUMENTS_DELTA.value: yield FunctionCallEvent( - event_type="function_call", service_type=event.type, function_call=FunctionCallContent( id=event.item_id, @@ -114,7 +114,7 @@ async def _parse_event(self, event: RealtimeServerEvent) -> AsyncGenerator[Realt # we put all event in the output buffer, but after the interpreted one. # so when dealing with them, make sure to check the type of the event, since they # might be of different types. - yield ServiceEvent(event_type="service", service_type=event.type, event=event) + yield ServiceEvent(service_type=event.type, event=event) @override async def update_session( @@ -137,7 +137,6 @@ async def update_session( ) await self.send( ServiceEvent( - event_type="service", service_type=SendEvents.SESSION_UPDATE, event={"settings": self._current_settings}, ) @@ -163,20 +162,6 @@ async def update_session( if create_response: await self.send(ServiceEvent(service_type=SendEvents.RESPONSE_CREATE)) - @override - def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: - from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa - OpenAIRealtimeExecutionSettings, - ) - - return OpenAIRealtimeExecutionSettings - - @override - def _update_function_choice_settings_callback( - self, - ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: - return update_settings_from_function_call_configuration - async def _parse_function_call_arguments_done( self, event: ResponseFunctionCallArgumentsDoneEvent, @@ -226,13 +211,13 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: match event.event_type: case "audio": await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=SendEvents.INPUT_AUDIO_BUFFER_APPEND, audio=event.audio.to_base64_bytestring() ) ) case "text": await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=SendEvents.CONVERSATION_ITEM_CREATE, **dict( type="message", @@ -248,7 +233,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: ) case "function_call": await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=SendEvents.CONVERSATION_ITEM_CREATE, **dict( type="function_call", @@ -264,7 +249,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: ) case "function_result": await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=SendEvents.CONVERSATION_ITEM_CREATE, **dict( type="function_call_output", @@ -281,13 +266,22 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: logger.error("Event data is empty") return settings = data.get("settings", None) - if not settings or not isinstance(settings, PromptExecutionSettings): + if not settings: logger.error("Event data does not contain 'settings'") return + if not isinstance(settings, OpenAIRealtimeExecutionSettings): + try: + settings = self.get_prompt_execution_settings_from_settings(settings) + except Exception as e: + logger.error( + f"Failed to properly create settings from passed settings: {settings}, error: {e}" + ) + return + assert isinstance(settings, OpenAIRealtimeExecutionSettings) # nosec if not settings.ai_model_id: settings.ai_model_id = self.ai_model_id await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, session=settings.prepare_settings_dict(), ) @@ -297,15 +291,15 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: logger.error("Event data does not contain 'audio'") return await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, audio=data["audio"], ) ) case SendEvents.INPUT_AUDIO_BUFFER_COMMIT: - await self._send(_create_realtime_client_event(event_type=event.service_type)) + await self._send(_create_openai_realtime_client_event(event_type=event.service_type)) case SendEvents.INPUT_AUDIO_BUFFER_CLEAR: - await self._send(_create_realtime_client_event(event_type=event.service_type)) + await self._send(_create_openai_realtime_client_event(event_type=event.service_type)) case SendEvents.CONVERSATION_ITEM_CREATE: if not data or "item" not in data: logger.error("Event data does not contain 'item'") @@ -316,7 +310,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: match item: case TextContent(): await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, **dict( type="message", @@ -332,7 +326,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: ) case FunctionCallContent(): await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, **dict( type="function_call", @@ -349,7 +343,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: case FunctionResultContent(): await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, **dict( type="function_call_output", @@ -363,7 +357,7 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: logger.error("Event data does not contain 'item_id'") return await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, item_id=data["item_id"], content_index=0, @@ -375,21 +369,52 @@ async def send(self, event: RealtimeEvent, **kwargs: Any) -> None: logger.error("Event data does not contain 'item_id'") return await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, item_id=data["item_id"], ) ) case SendEvents.RESPONSE_CREATE: await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, event_id=data.get("event_id", None) if data else None ) ) case SendEvents.RESPONSE_CANCEL: await self._send( - _create_realtime_client_event( + _create_openai_realtime_client_event( event_type=event.service_type, response_id=data.get("response_id", None) if data else None, ) ) + + @override + def get_prompt_execution_settings_class(self) -> type["PromptExecutionSettings"]: + from semantic_kernel.connectors.ai.open_ai.prompt_execution_settings.open_ai_realtime_execution_settings import ( # noqa + OpenAIRealtimeExecutionSettings, + ) + + return OpenAIRealtimeExecutionSettings + + @override + def _update_function_choice_settings_callback( + self, + ) -> Callable[[FunctionCallChoiceConfiguration, "PromptExecutionSettings", FunctionChoiceType], None]: + return update_settings_from_function_call_configuration + + @override + async def create_session( + self, + settings: "PromptExecutionSettings | None" = None, + chat_history: "ChatHistory | None" = None, + **kwargs: Any, + ) -> None: + pass + + @override + def receive(self, **kwargs: Any) -> AsyncGenerator[RealtimeEvent, None]: + pass + + @override + async def close_session(self) -> None: + pass diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py index 731ff423011b..2a6bf71dfd68 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_webrtc.py @@ -26,13 +26,12 @@ from openai._models import construct_type_unchecked from openai.types.beta.realtime.realtime_client_event import RealtimeClientEvent from openai.types.beta.realtime.realtime_server_event import RealtimeServerEvent -from pydantic import Field, PrivateAttr +from pydantic import PrivateAttr from semantic_kernel.connectors.ai.open_ai.services.realtime.const import ListenEvents from semantic_kernel.connectors.ai.open_ai.services.realtime.open_ai_realtime_base import OpenAIRealtimeBase -from semantic_kernel.connectors.ai.realtime_client_base import RealtimeEvent -from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioTrack from semantic_kernel.contents.audio_content import AudioContent +from semantic_kernel.contents.events import RealtimeEvent from semantic_kernel.contents.events.realtime_event import AudioEvent from semantic_kernel.utils.experimental_decorator import experimental_class @@ -51,7 +50,7 @@ class OpenAIRealtimeWebRTCBase(OpenAIRealtimeBase): protocol: ClassVar[Literal["webrtc"]] = "webrtc" peer_connection: RTCPeerConnection | None = None data_channel: RTCDataChannel | None = None - audio_track: MediaStreamTrack = Field(default_factory=SKAudioTrack) + audio_track: MediaStreamTrack | None = None _receive_buffer: asyncio.Queue[RealtimeEvent] = PrivateAttr(default_factory=asyncio.Queue) @override @@ -82,6 +81,8 @@ async def create_session( **kwargs: Any, ) -> None: """Create a session in the service.""" + if not self.audio_track: + raise Exception("Audio track not initialized") self.peer_connection = RTCPeerConnection( configuration=RTCConfiguration(iceServers=[RTCIceServer(urls="stun:stun.l.google.com:19302")]) ) @@ -161,8 +162,8 @@ async def _on_track(self, track: "MediaStreamTrack") -> None: try: await self._receive_buffer.put( AudioEvent( - service_type=ListenEvents.RESPONSE_AUDIO_DELTA, audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16", inner_content=frame), + service_type=ListenEvents.RESPONSE_AUDIO_DELTA, ), ) except Exception as e: diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py index 8adee40db02d..3b476d96d3c0 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/open_ai_realtime_websocket.py @@ -49,15 +49,13 @@ async def receive( async for event in self.connection: if event.type == ListenEvents.RESPONSE_AUDIO_DELTA.value: + audio_bytes = base64.b64decode(event.delta) if self.audio_output_callback: - await self.audio_output_callback(np.frombuffer(base64.b64decode(event.delta), dtype=np.int16)) + await self.audio_output_callback(np.frombuffer(audio_bytes, dtype=np.int16)) try: yield AudioEvent( - audio=AudioContent( - data=base64.b64decode(event.delta), - data_format="base64", - inner_content=event, - ), + audio=AudioContent(data=audio_bytes, data_format="base64", inner_content=event), + service_type=event.type, ) except Exception as e: logger.error(f"Error processing remote audio frame: {e!s}") diff --git a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py index a33531ca19c7..bb815eead6dd 100644 --- a/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py +++ b/python/semantic_kernel/connectors/ai/open_ai/services/realtime/utils.py @@ -65,7 +65,7 @@ def kernel_function_metadata_to_function_call_format( } -def _create_realtime_client_event(event_type: SendEvents, **kwargs: Any) -> RealtimeClientEvent: +def _create_openai_realtime_client_event(event_type: SendEvents, **kwargs: Any) -> RealtimeClientEvent: match event_type: case SendEvents.SESSION_UPDATE: return SessionUpdateEvent( diff --git a/python/semantic_kernel/connectors/ai/realtime_client_base.py b/python/semantic_kernel/connectors/ai/realtime_client_base.py index 0ad1fc13a089..cc70df1f3c90 100644 --- a/python/semantic_kernel/connectors/ai/realtime_client_base.py +++ b/python/semantic_kernel/connectors/ai/realtime_client_base.py @@ -2,7 +2,7 @@ import sys from abc import ABC, abstractmethod -from collections.abc import AsyncGenerator, Callable +from collections.abc import AsyncGenerator, Callable, Coroutine from typing import TYPE_CHECKING, Any, ClassVar if sys.version_info >= (3, 11): @@ -10,6 +10,8 @@ else: from typing_extensions import Self # pragma: no cover +from numpy import ndarray + from semantic_kernel.connectors.ai.function_call_choice_configuration import FunctionCallChoiceConfiguration from semantic_kernel.connectors.ai.function_choice_behavior import FunctionChoiceType from semantic_kernel.contents.events.realtime_event import RealtimeEvent @@ -26,6 +28,7 @@ class RealtimeClientBase(AIServiceClientBase, ABC): """Base class for a realtime client.""" SUPPORTS_FUNCTION_CALLING: ClassVar[bool] = False + audio_output_callback: Callable[[ndarray], Coroutine[Any, Any, None]] | None = None @abstractmethod async def send(self, event: RealtimeEvent) -> None: diff --git a/python/semantic_kernel/connectors/ai/utils/__init__.py b/python/semantic_kernel/connectors/ai/utils/__init__.py deleted file mode 100644 index 2cd59106a8a0..000000000000 --- a/python/semantic_kernel/connectors/ai/utils/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -from semantic_kernel.connectors.ai.utils.realtime_helpers import SKAudioPlayer, SKAudioTrack - -__all__ = ["SKAudioPlayer", "SKAudioTrack"] diff --git a/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py b/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py deleted file mode 100644 index 33fd09ce7f66..000000000000 --- a/python/semantic_kernel/connectors/ai/utils/realtime_helpers.py +++ /dev/null @@ -1,267 +0,0 @@ -# Copyright (c) Microsoft. All rights reserved. - -import asyncio -import logging -from contextlib import asynccontextmanager -from typing import Any, ClassVar, Final - -import numpy as np -import numpy.typing as npt -from aiortc.mediastreams import MediaStreamError, MediaStreamTrack -from av.audio.frame import AudioFrame -from av.frame import Frame -from pydantic import PrivateAttr -from sounddevice import InputStream, OutputStream - -from semantic_kernel.connectors.ai.realtime_client_base import RealtimeClientBase -from semantic_kernel.contents.audio_content import AudioContent -from semantic_kernel.contents.events.realtime_event import AudioEvent -from semantic_kernel.kernel_pydantic import KernelBaseModel - -logger = logging.getLogger(__name__) - -SAMPLE_RATE: Final[int] = 48000 -TRACK_CHANNELS: Final[int] = 1 -PLAYER_CHANNELS: Final[int] = 2 -FRAME_DURATION: Final[int] = 20 -DTYPE: Final[npt.DTypeLike] = np.int16 - - -class SKAudioTrack(KernelBaseModel, MediaStreamTrack): - """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice.""" - - kind: ClassVar[str] = "audio" - device: str | int | None = None - sample_rate: int = SAMPLE_RATE - channels: int = TRACK_CHANNELS - frame_duration: int = FRAME_DURATION - dtype: npt.DTypeLike = DTYPE - frame_size: int = 0 - _queue: asyncio.Queue[Frame] = PrivateAttr(default_factory=asyncio.Queue) - _is_recording: bool = False - _stream: InputStream | None = None - _recording_task: asyncio.Task | None = None - _loop: asyncio.AbstractEventLoop | None = None - _pts: int = 0 - - def __init__( - self, - *, - device: str | int | None = None, - sample_rate: int = SAMPLE_RATE, - channels: int = TRACK_CHANNELS, - frame_duration: int = FRAME_DURATION, - dtype: npt.DTypeLike = DTYPE, - ): - """A simple class that implements the WebRTC MediaStreamTrack for audio from sounddevice. - - Make sure the device is set to the correct device for your system. - - Args: - device: The device id to use for recording audio. - sample_rate: The sample rate for the audio. - channels: The number of channels for the audio. - frame_duration: The duration of each audio frame in milliseconds. - dtype: The data type for the audio. - **kwargs: Additional keyword arguments. - """ - args = { - "device": device, - "sample_rate": sample_rate, - "channels": channels, - "frame_duration": frame_duration, - "dtype": dtype, - } - args["frame_size"] = int( - args.get("sample_rate", SAMPLE_RATE) * args.get("frame_duration", FRAME_DURATION) / 1000 - ) - super().__init__(**args) - MediaStreamTrack.__init__(self) - - async def recv(self) -> Frame: - """Receive the next frame of audio data.""" - if not self._recording_task: - self._recording_task = asyncio.create_task(self.start_recording()) - - try: - frame = await self._queue.get() - self._queue.task_done() - return frame - except Exception as e: - logger.error(f"Error receiving audio frame: {e!s}") - raise MediaStreamError("Failed to receive audio frame") - - @asynccontextmanager - async def stream_to_realtime_client(self, realtime_client: RealtimeClientBase): - """Stream audio data to a RealtimeClientBase.""" - while True: - frame = await self.recv() - await realtime_client.send(AudioEvent(audio=AudioContent(data=frame.to_ndarray(), data_format="np.int16"))) - yield - await asyncio.sleep(0.01) - - def _sounddevice_callback(self, indata: np.ndarray, frames: int, time: Any, status: Any) -> None: - if status: - logger.warning(f"Audio input status: {status}") - if self._loop and self._loop.is_running(): - asyncio.run_coroutine_threadsafe(self._queue.put(self._create_frame(indata)), self._loop) - - def _create_frame(self, indata: np.ndarray) -> Frame: - audio_data = indata.copy() - if audio_data.dtype != self.dtype: - audio_data = ( - (audio_data * 32767).astype(self.dtype) if self.dtype == np.int16 else audio_data.astype(self.dtype) - ) - frame = AudioFrame( - format="s16", - layout="mono", - samples=len(audio_data), - ) - frame.rate = self.sample_rate - frame.pts = self._pts - frame.planes[0].update(audio_data.tobytes()) - self._pts += len(audio_data) - return frame - - async def start_recording(self): - """Start recording audio from the input device.""" - if self._is_recording: - return - - self._is_recording = True - self._loop = asyncio.get_running_loop() - self._pts = 0 # Reset pts when starting recording - - try: - self._stream = InputStream( - device=self.device, - channels=self.channels, - samplerate=self.sample_rate, - dtype=self.dtype, - blocksize=self.frame_size, - callback=self._sounddevice_callback, - ) - self._stream.start() - - while self._is_recording: - await asyncio.sleep(0.1) - - except Exception as e: - logger.error(f"Error in audio recording: {e!s}") - raise - finally: - self._is_recording = False - - -class SKAudioPlayer(KernelBaseModel): - """Simple class that plays audio using sounddevice. - - Make sure the device_id is set to the correct device for your system. - - The sample rate, channels and frame duration - should be set to match the audio you - are receiving. - - Args: - device: The device id to use for playing audio. - sample_rate: The sample rate for the audio. - channels: The number of channels for the audio. - dtype: The data type for the audio. - frame_duration: The duration of each audio frame in milliseconds - - """ - - device: int | None = None - sample_rate: int = SAMPLE_RATE - channels: int = PLAYER_CHANNELS - dtype: npt.DTypeLike = DTYPE - frame_duration: int = FRAME_DURATION - _queue: asyncio.Queue[np.ndarray] | None = PrivateAttr(default=None) - _stream: OutputStream | None = PrivateAttr(default=None) - - async def __aenter__(self): - """Start the audio stream when entering a context.""" - self.start() - return self - - async def __aexit__(self, exc_type, exc, tb): - """Stop the audio stream when exiting a context.""" - self.stop() - - def start(self): - """Start the audio stream.""" - self._queue = asyncio.Queue() - self._stream = OutputStream( - callback=self._sounddevice_callback, - samplerate=self.sample_rate, - channels=self.channels, - dtype=self.dtype, - blocksize=int(self.sample_rate * self.frame_duration / 1000), - device=self.device, - ) - if self._stream and self._queue: - self._stream.start() - - def stop(self): - """Stop the audio stream.""" - if self._stream: - self._stream.stop() - self._stream = None - self._queue = None - - def _sounddevice_callback(self, outdata, frames, time, status): - """This callback is called by sounddevice when it needs more audio data to play.""" - if status: - logger.info(f"Audio output status: {status}") - if self._queue: - if self._queue.empty(): - return - data: np.ndarray = self._queue.get_nowait() - if data.size == frames: - outdata[:] = data.reshape(outdata.shape) - self._queue.task_done() - else: - if data.size > frames: - self._queue.put_nowait(data[frames:]) - outdata[:] = np.concatenate((np.empty(0, dtype=np.int16), data[:frames])).reshape(outdata.shape) - else: - outdata[:] = np.concatenate((data, np.zeros(frames - len(data), dtype=np.int16))).reshape( - outdata.shape - ) - self._queue.task_done() - - async def client_callback(self, content: np.ndarray): - """This function can be passed to the audio_output_callback field of the RealtimeClientBase.""" - if self._queue: - await self._queue.put(content) - else: - logger.error( - "Audio queue not initialized, make sure to call start before " - "using the player, or use the context manager." - ) - - async def add_audio(self, audio_content: AudioContent) -> None: - """This function is used to add audio to the queue for playing. - - It first checks if there is a AudioFrame in the inner_content of the AudioContent. - If not, it checks if the data is a numpy array, bytes, or a string and converts it to a numpy array. - """ - if not self._queue: - logger.error( - "Audio queue not initialized, make sure to call start before " - "using the player, or use the context manager." - ) - return - if audio_content.inner_content and isinstance(audio_content.inner_content, AudioFrame): - await self._queue.put(audio_content.inner_content.to_ndarray()) - return - if isinstance(audio_content.data, np.ndarray): - await self._queue.put(audio_content.data) - return - if isinstance(audio_content.data, bytes): - await self._queue.put(np.frombuffer(audio_content.data, dtype=self.dtype)) - return - if isinstance(audio_content.data, str): - await self._queue.put(np.frombuffer(audio_content.data.encode(), dtype=self.dtype)) - return - logger.error(f"Unknown audio content: {audio_content}") diff --git a/python/semantic_kernel/contents/events/realtime_event.py b/python/semantic_kernel/contents/events/realtime_event.py index edb2c5917778..682c3b4d4e79 100644 --- a/python/semantic_kernel/contents/events/realtime_event.py +++ b/python/semantic_kernel/contents/events/realtime_event.py @@ -1,17 +1,18 @@ # Copyright (c) Microsoft. All rights reserved. -from typing import Annotated, Any, Literal, TypeAlias, Union +from typing import Annotated, Any, ClassVar, Literal, TypeAlias, Union from pydantic import Field from semantic_kernel.contents.audio_content import AudioContent from semantic_kernel.contents.function_call_content import FunctionCallContent from semantic_kernel.contents.function_result_content import FunctionResultContent +from semantic_kernel.contents.image_content import ImageContent from semantic_kernel.contents.text_content import TextContent from semantic_kernel.kernel_pydantic import KernelBaseModel RealtimeEvent: TypeAlias = Annotated[ - Union["ServiceEvent", "AudioEvent", "TextEvent", "FunctionCallEvent", "FunctionResultEvent"], + Union["ServiceEvent", "AudioEvent", "TextEvent", "FunctionCallEvent", "FunctionResultEvent", "ImageEvent"], Field(discriminator="event_type"), ] @@ -19,38 +20,46 @@ class ServiceEvent(KernelBaseModel): """Base class for all service events.""" - event_type: Literal["service"] = "service" + event: Any | None = Field(default=None, description="The event content.") service_type: str - event: Any | None = None + event_type: ClassVar[Literal["service"]] = "service" class AudioEvent(KernelBaseModel): """Audio event type.""" - event_type: Literal["audio"] = "audio" + audio: AudioContent = Field(..., description="Audio content.") service_type: str | None = None - audio: AudioContent + event_type: ClassVar[Literal["audio"]] = "audio" class TextEvent(KernelBaseModel): """Text event type.""" - event_type: Literal["text"] = "text" + text: TextContent = Field(..., description="Text content.") service_type: str | None = None - text: TextContent + event_type: ClassVar[Literal["text"]] = "text" class FunctionCallEvent(KernelBaseModel): """Function call event type.""" - event_type: Literal["function_call"] = "function_call" + function_call: FunctionCallContent = Field(..., description="Function call content.") service_type: str | None = None - function_call: FunctionCallContent + event_type: ClassVar[Literal["function_call"]] = "function_call" class FunctionResultEvent(KernelBaseModel): """Function result event type.""" - event_type: Literal["function_result"] = "function_result" + function_result: FunctionResultContent = Field(..., description="Function result content.") service_type: str | None = None - function_result: FunctionResultContent + event_type: ClassVar[Literal["function_result"]] = "function_result" + + +class ImageEvent(KernelBaseModel): + """Image event type.""" + + image: ImageContent = Field(..., description="Image content.") + service_type: str | None = None + event_type: ClassVar[Literal["image"]] = "image" From f94b631b24d3a8bdb3b189f2c80945734f66c612 Mon Sep 17 00:00:00 2001 From: eavanvalkenburg Date: Fri, 31 Jan 2025 15:54:56 +0100 Subject: [PATCH 25/25] add image event import --- python/semantic_kernel/contents/events/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/python/semantic_kernel/contents/events/__init__.py b/python/semantic_kernel/contents/events/__init__.py index 7466a652364b..432c4a9c0851 100644 --- a/python/semantic_kernel/contents/events/__init__.py +++ b/python/semantic_kernel/contents/events/__init__.py @@ -4,6 +4,7 @@ AudioEvent, FunctionCallEvent, FunctionResultEvent, + ImageEvent, RealtimeEvent, ServiceEvent, TextEvent, @@ -13,6 +14,7 @@ "AudioEvent", "FunctionCallEvent", "FunctionResultEvent", + "ImageEvent", "RealtimeEvent", "ServiceEvent", "TextEvent",