Update UI and add Proactive Video feature to Websocket-demo-app

.github/CODEOWNERS

@@ -24,7 +24,7 @@
 /generative-ai/gemini/mcp/intro_to_mcp.ipynb                                                                                     @GoogleCloudPlatform/generative-ai-devrel @wadave
 /generative-ai/gemini/model-optimizer/intro_model_optimizer.ipynb                                                                @GoogleCloudPlatform/generative-ai-devrel @nardosalemu
 /generative-ai/gemini/multimodal-dataset                                                                                         @GoogleCloudPlatform/generative-ai-devrel @fthoele @cleop-google @diskontinuum
-/generative-ai/gemini/multimodal-live-api/websocket-demo-app/                                                                    @GoogleCloudPlatform/generative-ai-devrel @ZackAkil
+/generative-ai/gemini/multimodal-live-api/websocket-demo-app/                                                                    @GoogleCloudPlatform/generative-ai-devrel @ZackAkil @tpareek18
 /generative-ai/gemini/rag-engine                                                                                                 @GoogleCloudPlatform/generative-ai-devrel @edtsoi430
 /generative-ai/gemini/responsible-ai                                                                                             @GoogleCloudPlatform/generative-ai-devrel @iamthuya
 /generative-ai/gemini/responsible-ai/gemini_safety_ratings.ipynb                                                                 @GoogleCloudPlatform/generative-ai-devrel @ghchinoy

gemini/multimodal-live-api/websocket-demo-app/.prettierrc

@@ -1,4 +1 @@
-{
-    "bracketSameLine": true,
-    "tabWidth": 4
-}
+{ "bracketSameLine": true, "tabWidth": 4 }

gemini/multimodal-live-api/websocket-demo-app/backend/main.py

@@ -5,9 +5,11 @@
 from websockets.legacy.protocol import WebSocketCommonProtocol
 from websockets.legacy.server import WebSocketServerProtocol
 
+# !!! Need to change this as well to make the environment switch work.
+# HOST = "us-central1-autopush-aiplatform.sandbox.googleapis.com"
+# HOST = "us-central1-staging-aiplatform.sandbox.googleapis.com"
 HOST = "us-central1-aiplatform.googleapis.com"
 SERVICE_URL = f"wss://{HOST}/ws/google.cloud.aiplatform.v1beta1.LlmBidiService/BidiGenerateContent"
-
 DEBUG = False
 
 

gemini/multimodal-live-api/websocket-demo-app/frontend/gemini-live-api.js

@@ -3,17 +3,43 @@
         this.data = "";
         this.type = "";
         this.endOfTurn = data?.serverContent?.turnComplete;
+        this.interrupt = data?.serverContent?.interrupted;
 
         const parts = data?.serverContent?.modelTurn?.parts;
 
         if (data?.setupComplete) {
             this.type = "SETUP COMPLETE";
+        } else if (data?.voiceActivityDetectionSignal) {
+            this.type = "VAD_SIGNAL";
         } else if (parts?.length && parts[0].text) {
             this.data = parts[0].text;
             this.type = "TEXT";
         } else if (parts?.length && parts[0].inlineData) {
             this.data = parts[0].inlineData.data;
             this.type = "AUDIO";
+        } else if (data?.sessionResumptionUpdate) {
+            this.type = "RESUMPTION";
+            this.data = data?.sessionResumptionUpdate?.newHandle;
+        } else if (data?.serverContent?.inputTranscription) {
+            this.type = "INPUT_TRANSCRIPTION";
+            if (data?.serverContent?.inputTranscription?.text) {
+                this.data = data?.serverContent?.inputTranscription?.text;
+            } else if (data?.serverContent?.inputTranscription?.finished) {
+                this.data = "Finished Inp Transcription: " + data?.serverContent?.inputTranscription?.finished
+            }
+        } else if (data?.serverContent?.outputTranscription) {
+            this.type = "OUTPUT_TRANSCRIPTION";
+            if (data?.serverContent?.outputTranscription?.text) {
+                this.data = data?.serverContent?.outputTranscription?.text;
+            } else if (data?.serverContent?.outputTranscription?.finished) {
+                this.data = "Finished Out Transcription: " + data?.serverContent?.outputTranscription?.finished
+            }
+        } else if (this.endOfTurn) {
+            this.data = "END OF TURN";
+            this.type = "END_OF_TURN";
+        } else if (this.interrupt) {
+            this.data = "INTERRUPT";
+            this.type = "INTERRUPT";
         }
     }
 }
@@ -25,6 +51,7 @@
         this.projectId = projectId;
         this.model = model;
         this.modelUri = `projects/${this.projectId}/locations/us-central1/publishers/google/models/${this.model}`;
+        console.log("Model URI:", this.modelUri);
 
         this.responseModalities = ["AUDIO"];
         this.systemInstructions = "";
@@ -47,6 +74,18 @@
         this.accessToken = "";
         this.websocket = null;
 
+        this.enableInputTranscript = false;
+        this.enableOutputTranscript = false;
+        this.voiceName = "";
+        this.voiceLocale = "";
+        this.enableSessionResumption = false;
+        this.resumptionHandle = "";
+        this.disableDetection = false;
+        this.disableInterruption = false;
+        this.startSensitivity = "";
+        this.endSensitivity = "";
+        this.enableProactiveVideo = false;
+
         console.log("Created Gemini Live API object: ", this);
     }
 
@@ -55,11 +94,42 @@
         this.modelUri = `projects/${this.projectId}/locations/us-central1/publishers/google/models/${this.model}`;
     }
 
+    setApiHost(apiHost) {
+        this.apiHost = apiHost;
+    }
+
     setAccessToken(newAccessToken) {
         console.log("setting access token: ", newAccessToken);
         this.accessToken = newAccessToken;
     }
 
+    setTranscript(input, output) {
+        console.log("input transcript: ", input, "output transcript: ", output);
+        this.enableInputTranscript = input;
+        this.enableOutputTranscript = output;
+    }
+
+    setVoice(name, locale) {
+        this.voiceName = name;
+        this.voiceLocale = locale;
+    }
+
+    setResumption(enable, handle) {
+        this.enableSessionResumption = enable;
+        this.resumptionHandle = handle;
+    }
+
+    setVad(disableInterruption, disableDetection, startSen, endSen) {
+        this.disableDetection = disableDetection;
+        this.disableInterruption = disableInterruption;
+        this.startSensitivity = startSen;
+        this.endSensitivity = endSen;
+    }
+
+    setProactiveVideo(enable) {
+        this.enableProactiveVideo = enable;
+    }
+
     connect(accessToken) {
         this.setAccessToken(accessToken);
         this.setupWebSocketToService();
@@ -115,14 +185,69 @@
         const sessionSetupMessage = {
             setup: {
                 model: this.modelUri,
+                realtime_input_config: {},
+                explicit_vad_signal: true,
                 generation_config: {
                     response_modalities: this.responseModalities,
-                },
-                system_instruction: {
-                    parts: [{ text: this.systemInstructions }],
+                    speech_config: {
+                        voice_config: {
+                            prebuilt_voice_config: { voice_name: this.voiceName },
+                        },
+                        language_code: this.voiceLocale
+                    }
                 },
             },
         };
+
+        if (this.systemInstructions && this.systemInstructions.trim()) {
+            sessionSetupMessage.setup.system_instruction = {
+                parts: [{ text: this.systemInstructions }],
+            };
+        }
+
+        if (this.enableInputTranscript) {
+            sessionSetupMessage.setup.input_audio_transcription = {};
+        }
+        if (this.enableOutputTranscript) {
+            sessionSetupMessage.setup.output_audio_transcription = {};
+        }
+        if (this.enableSessionResumption) {
+            sessionSetupMessage.setup.session_resumption = {
+                handle: this.resumptionHandle
+            };
+        }
+
+        sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection = {}
+        if (this.disableDetection) {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.disabled = true;
+        }
+        if (this.disableInterruption) {
+            sessionSetupMessage.setup.realtime_input_config.activity_handling = 2;
+        }
+
+        if (this.startSensitivity === "") {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.start_of_speech_sensitivity = 0;
+        } else if (this.startSensitivity === "low") {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.start_of_speech_sensitivity = 2;
+        } else if (this.startSensitivity === "high") {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.start_of_speech_sensitivity = 1;
+        }
+
+        if (this.endSensitivity === "") {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.end_of_speech_sensitivity = 0;
+        } else if (this.endSensitivity === "low") {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.end_of_speech_sensitivity = 2;
+        } else if (this.endSensitivity === "high") {
+            sessionSetupMessage.setup.realtime_input_config.automatic_activity_detection.end_of_speech_sensitivity = 1;
+        }
+
+        if (this.enableProactiveVideo) {
+            sessionSetupMessage.setup.proactivity = {
+                proactive_video: true,
+            };
+        }
+
+        console.log("setup message: ", sessionSetupMessage);
         this.sendMessage(sessionSetupMessage);
     }
 
@@ -141,6 +266,24 @@
         this.sendMessage(textMessage);
     }
 
+    sendVoiceActivityMessage(start) {
+        if (start) {
+            const startMessage = {
+                realtime_input: {
+                    activity_start: {},
+                },
+            };
+            this.sendMessage(startMessage);
+        } else {
+            const endMessage = {
+                realtime_input: {
+                    activity_end: {},
+                },
+            };
+            this.sendMessage(endMessage);
+        }
+    }
+
     sendRealtimeInputMessage(data, mime_type) {
         const message = {
             realtime_input: {