From bcce78b273599222ab98146c1da23747198be694 Mon Sep 17 00:00:00 2001
From: akdeb <deb2@illinois.edu>
Date: Wed, 11 Jun 2025 15:22:51 +0100
Subject: [PATCH] both openai and gemini working well

---
 server-deno/main.ts          | 13 ++++++++++++
 server-deno/models/gemini.ts | 41 +++++++++---------------------------
 server-deno/models/openai.ts | 12 -----------
 3 files changed, 23 insertions(+), 43 deletions(-)

diff --git a/server-deno/main.ts b/server-deno/main.ts
index 4866ef2..dab0dc5 100644
--- a/server-deno/main.ts
+++ b/server-deno/main.ts
@@ -43,6 +43,19 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
     const systemPrompt = createSystemPrompt(chatHistory, payload);
 
     const provider = user.personality?.provider;
+
+    // send user details to client
+    // when DEV_MODE is true, we send the default values 100, false, false
+    ws.send(
+        JSON.stringify({
+            type: "auth",
+            volume_control: user.device?.volume ?? 20,
+            is_ota: user.device?.is_ota ?? false,
+            is_reset: user.device?.is_reset ?? false,
+            pitch_factor: user.personality?.pitch_factor ?? 1,
+        }),
+    );
+
     switch (provider) {
         case "openai":
             await connectToOpenAI(
diff --git a/server-deno/models/gemini.ts b/server-deno/models/gemini.ts
index d0d5ac1..c38ebac 100644
--- a/server-deno/models/gemini.ts
+++ b/server-deno/models/gemini.ts
@@ -19,28 +19,7 @@ export const connectToGemini = async (
 	systemPrompt: string,
 ) => {
 	const { user, supabase } = payload;
-	const { oai_voice, pitch_factor } = user.personality ?? {
-		oai_voice: "Sadachbia",
-		provider: "gemini",
-		pitch_factor: 1,
-	};
-
-	const { is_ota, is_reset, volume } = user.device ?? {
-		is_ota: false,
-		is_reset: false,
-		volume: 10,
-	};
-
-	// Send user details to client
-	ws.send(
-		JSON.stringify({
-			type: "auth",
-			volume_control: volume,
-			is_ota: is_ota,
-			is_reset: is_reset,
-			pitch_factor: pitch_factor,
-		}),
-	);
+	const { oai_voice } = user.personality ?? { oai_voice: "Sadachbia" };
 
 	console.log(`Connecting with Gemini key "${geminiApiKey.slice(0, 3)}..."`);
 
@@ -60,7 +39,7 @@ export const connectToGemini = async (
 		realtimeInputConfig: {
 			automaticActivityDetection: {
 				disabled: false, // Keep VAD enabled
-				endOfSpeechSensitivity: EndSensitivity.END_SENSITIVITY_HIGH, // How sensitive to detect speech ending
+				endOfSpeechSensitivity: EndSensitivity.END_SENSITIVITY_LOW, // How sensitive to detect speech ending
 				silenceDurationMs: 100, // How much silence before considering speech ended
 			},
 		},
@@ -104,14 +83,14 @@ export const connectToGemini = async (
 					done = true;
 				}
 
-				if (message.serverContent.turnComplete) {
-					ws.send(
-						JSON.stringify({
-							type: "server",
-							msg: "AUDIO.COMMITTED",
-						}),
-					);
-				}
+				// if (message.serverContent.turnComplete) {
+				// 	ws.send(
+				// 		JSON.stringify({
+				// 			type: "server",
+				// 			msg: "AUDIO.COMMITTED",
+				// 		}),
+				// 	);
+				// }
 			}
 		}
 		return turns;
diff --git a/server-deno/models/openai.ts b/server-deno/models/openai.ts
index d7be29d..cbc1664 100644
--- a/server-deno/models/openai.ts
+++ b/server-deno/models/openai.ts
@@ -41,18 +41,6 @@ export const connectToOpenAI = async (
 ) => {
 	const { user, supabase } = payload;
 
-	// send user details to client
-	// when DEV_MODE is true, we send the default values 100, false, false
-	ws.send(
-		JSON.stringify({
-			type: "auth",
-			volume_control: user.device?.volume ?? 20,
-			is_ota: user.device?.is_ota ?? false,
-			is_reset: user.device?.is_reset ?? false,
-			pitch_factor: user.personality?.pitch_factor ?? 1,
-		}),
-	);
-
 	let currentItemId: string | null = null;
 	let currentCallId: string | null = null;