fixing eleven labs api

firmware-arduino/src/Config.cpp

@@ -33,11 +33,11 @@ volatile bool sleepRequested = false;
  */
 
 #ifdef DEV_MODE
-const char *ws_server = "192.168.1.155";
+const char *ws_server = "192.168.1.121";
 const uint16_t ws_port = 8000;
 const char *ws_path = "/";
 // Backend server details 
-const char *backend_server = "192.168.1.155";
+const char *backend_server = "192.168.1.121";
 const uint16_t backend_port = 3000;
 
 #elif defined(PROD_MODE)

server-deno/main.ts

@@ -4,7 +4,7 @@ import type {
     WebSocket as WSWebSocket,
     WebSocketServer as _WebSocketServer,
 } from "npm:@types/ws";
-import { authenticateUser, elevenLabsApiKey } from "./utils.ts";
+import { authenticateUser } from "./utils.ts";
 import {
     createFirstMessage,
     createSystemPrompt,
@@ -59,7 +59,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
     ws.send(
         JSON.stringify({
             type: "auth",
-            volume_control: user.device?.volume ?? 20,
+            volume_control: user.device?.volume ?? 100,
             is_ota: user.device?.is_ota ?? false,
             is_reset: user.device?.is_reset ?? false,
             pitch_factor: user.personality?.pitch_factor ?? 1,
@@ -92,20 +92,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
             await connectToGrok(providerArgs);
             break;
         case "elevenlabs":
-            const agentId = user.personality?.oai_voice ?? "";
+            await connectToElevenLabs(providerArgs);
-            
-            if (!elevenLabsApiKey) {
-                throw new Error("ELEVENLABS_API_KEY environment variable is required");
-            }
-            
-            await connectToElevenLabs(
-                ws,
-                payload,
-                connectionPcmFile,
-                agentId,
-                elevenLabsApiKey,
-                closeHandler,
-            );
             break;
         case "hume":
             await connectToHume(providerArgs);

server-deno/models/elevenlabs.ts

@@ -7,9 +7,13 @@ import {
 	type IncomingSocketEvent,
 	type DisconnectionDetails
 } from "npm:@elevenlabs/client";
-
 import { addConversation, getDeviceInfo } from "../supabase.ts";
-import { createOpusPacketizer, isDev } from "../utils.ts";
+import {
+	createOpusPacketizer,
+	isDev,
+	elevenLabsApiKey,
+	SAMPLE_RATE,
+} from "../utils.ts";
 
 // Calculate audio level for debugging
 function calculateAudioLevel(audioData: any): number {
@@ -26,40 +30,83 @@ function calculateAudioLevel(audioData: any): number {
 	return Math.round(sum / samples.length);
 }
 
-export const connectToElevenLabs = async (
+// Resample mono PCM16 little-endian audio with linear interpolation.
-    ws: WebSocket,
+function resamplePcm16Mono(
-    payload: IPayload,
+	inputBytes: Buffer,
-    connectionPcmFile: Deno.FsFile | null,
+	fromRate: number,
-    agentId: string,
+	toRate: number,
-    apiKey: string,
+): Buffer {
-    closeHandler: () => Promise<void>,
+	if (fromRate === toRate || inputBytes.length === 0) {
-) => {
+		return inputBytes;
-    console.log(apiKey, agentId);
+	}
-    const { user, supabase } = payload;
 
+	const inputSamples = inputBytes.length / 2;
+	const outputSamples = Math.max(1, Math.floor((inputSamples * toRate) / fromRate));
+	const output = Buffer.alloc(outputSamples * 2);
+
+	for (let i = 0; i < outputSamples; i++) {
+		const sourcePos = (i * fromRate) / toRate;
+		const leftIndex = Math.floor(sourcePos);
+		const rightIndex = Math.min(leftIndex + 1, inputSamples - 1);
+		const frac = sourcePos - leftIndex;
+
+		const left = inputBytes.readInt16LE(leftIndex * 2);
+		const right = inputBytes.readInt16LE(rightIndex * 2);
+		const sample = Math.round(left + (right - left) * frac);
+		output.writeInt16LE(sample, i * 2);
+	}
+
+	return output;
+}
+
+export const connectToElevenLabs = async ({
+	ws,
+	payload,
+	connectionPcmFile,
+	firstMessage,
+	closeHandler,
+}: ProviderArgs) => {
+	const agentId = payload.user.personality?.voice?.config?.config_id ??
+		payload.user.personality?.oai_voice;
+	const apiKey = elevenLabsApiKey;
+
+	if (!agentId || !apiKey) {
+		throw new Error("Agent ID or API key is missing");
+	}
+
+	const { user, supabase } = payload;
 	const opus = createOpusPacketizer((packet) => ws.send(packet));
 
-    // Queue messages until ElevenLabs connection is ready
+	// Queue messages until ElevenLabs connection is ready.
-    const messageQueue: RawData[] = [];
+	const messageQueue: Array<{ data: RawData; isBinary: boolean }> = [];
 	let isElevenLabsConnected = false;
 	let elevenLabsConnection: WebSocketConnection | null = null;
 	let hasResponseStarted = false;
+	let elevenInputSampleRate = 16000;
+	let elevenOutputSampleRate = 16000;
 
-    // Handle messages from ESP32 client
+	// Handle messages from ESP32 ws client.
 	const handleClientMessage = async (data: any, isBinary: boolean) => {
 		try {
 			if (isBinary) {
-                const base64Data = data.toString("base64");
-
 				if (isDev && connectionPcmFile) {
 					await connectionPcmFile.write(data);
 				}
 
-                // Send audio to ElevenLabs using their client
+				// Send audio to ElevenLabs using the expected input sample rate.
 				if (isElevenLabsConnected && elevenLabsConnection) {
-                    // Check if audio contains actual speech (simple volume check)
+					const sourceBuffer = Buffer.from(data);
+					const pcmForEleven = resamplePcm16Mono(
+						sourceBuffer,
+						SAMPLE_RATE,
+						elevenInputSampleRate,
+					);
+					const base64Data = pcmForEleven.toString("base64");
+
 					const audioLevel = calculateAudioLevel(data);
-                    console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`);
+					console.log(
+						`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, resampled=${pcmForEleven.length} bytes, inRate=${SAMPLE_RATE}, elevenInRate=${elevenInputSampleRate}, level=${audioLevel}`,
+					);
 
 					try {
 						elevenLabsConnection.sendMessage({
@@ -69,7 +116,9 @@ export const connectToElevenLabs = async (
 						console.error("Error sending audio to ElevenLabs:", error);
 					}
 				} else {
-                    console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`);
+					console.log(
+						`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`,
+					);
 				}
 			} else {
 				const message = JSON.parse(data.toString("utf-8"));
@@ -80,7 +129,7 @@ export const connectToElevenLabs = async (
 							console.log("Interrupt detected");
 							if (elevenLabsConnection) {
 								elevenLabsConnection.sendMessage({
-                                    type: "user_activity"
+									type: "user_activity",
 								});
 							}
 							break;
@@ -99,39 +148,70 @@ export const connectToElevenLabs = async (
 		}
 	};
 
+	// Register handlers immediately so early ESP32 audio is not dropped while ElevenLabs connects.
+	ws.on("message", (data: any, isBinary: boolean) => {
+		if (!isElevenLabsConnected) {
+			messageQueue.push({ data, isBinary });
+		} else {
+			handleClientMessage(data, isBinary);
+		}
+	});
+
+	ws.on("error", (error: any) => {
+		console.error("ESP32 WebSocket error:", error);
+		opus.close();
+		elevenLabsConnection?.close();
+	});
+
+	ws.on("close", async (code: number, reason: string) => {
+		console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
+		await closeHandler();
+		opus.close();
+		elevenLabsConnection?.close();
+
+		if (isDev && connectionPcmFile) {
+			connectionPcmFile.close();
+			console.log("Closed debug audio file.");
+		}
+	});
+
 	try {
-        // For server-side usage, we need to get a signed URL first
+		// For server-side usage, we need to get a signed URL first.
 		const signedUrlResponse = await fetch(
 			`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
 			{
 				headers: {
-                    'xi-api-key': apiKey,
+					"xi-api-key": apiKey,
+				},
 			},
-            }
 		);
 
 		if (!signedUrlResponse.ok) {
-            throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`);
+			throw new Error(
+				`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`,
+			);
 		}
 
 		const { signed_url } = await signedUrlResponse.json();
 
-        // Use default audio formats (let ElevenLabs auto-detect)
-        const modifiedSignedUrl = signed_url;
-
-        // Create ElevenLabs connection using signed URL for server-side usage
 		const sessionConfig: SessionConfig = {
-            signedUrl: modifiedSignedUrl,
+			signedUrl: signed_url,
 			connectionType: "websocket",
 		};
 
 		elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
+		elevenInputSampleRate = elevenLabsConnection.inputFormat.sampleRate;
+		elevenOutputSampleRate = elevenLabsConnection.outputFormat.sampleRate;
 
 		console.log("Connected to ElevenLabs successfully!");
+		console.log(
+			`ElevenLabs formats: input=${elevenLabsConnection.inputFormat.format}_${elevenInputSampleRate}, output=${elevenLabsConnection.outputFormat.format}_${elevenOutputSampleRate}`,
+		);
 		isElevenLabsConnected = true;
-        console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`);
+		console.log(
+			"ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK",
+		);
 
-        // Set up ElevenLabs event handlers
 		elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
 			console.log("ElevenLabs message type:", event);
 
@@ -141,40 +221,48 @@ export const connectToElevenLabs = async (
 					break;
 
 				case "ping":
-                    // Handle ping messages - send pong response
 					console.log("Received ping from ElevenLabs, sending pong");
 					if (event.ping_event?.event_id) {
-                        elevenLabsConnection.sendMessage({
+						elevenLabsConnection?.sendMessage({
 							type: "pong",
-                            event_id: event.ping_event.event_id
+							event_id: event.ping_event.event_id,
 						});
 					}
 					break;
 
 				case "audio":
 					if (event.audio_event?.audio_base_64) {
-                        // Send RESPONSE.CREATED before first audio chunk
 						if (!hasResponseStarted) {
-                            console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
+							console.log(
+								"Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
+							);
 							opus.reset();
 							ws.send(JSON.stringify({
 								type: "server",
-                                msg: "RESPONSE.CREATED"
+								msg: "RESPONSE.CREATED",
 							}));
 							hasResponseStarted = true;
 						}
 
-                        const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64");
+						const audioBuffer = Buffer.from(
-                        console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes`);
+							event.audio_event.audio_base_64,
-
+							"base64",
-                        // Use Opus packetizer to encode and send audio
+						);
-                        opus.push(audioBuffer);
+						const pcmForEsp32 = resamplePcm16Mono(
+							audioBuffer,
+							elevenOutputSampleRate,
+							SAMPLE_RATE,
+						);
+						opus.push(pcmForEsp32);
 					}
 					break;
 
 				case "user_transcript":
 					if (event.user_transcription_event?.user_transcript) {
-                        console.log("User transcript:", event.user_transcription_event.user_transcript);
+						console.log(
+							"User transcript:",
+							event.user_transcription_event.user_transcript,
+						);
 						addConversation(
 							supabase,
 							"user",
@@ -183,11 +271,12 @@ export const connectToElevenLabs = async (
 						);
 
 						if (!hasResponseStarted) {
-                            console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
+							console.log(
-                            opus.reset();
+								"Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
+							);
 							ws.send(JSON.stringify({
 								type: "server",
-                                msg: "RESPONSE.CREATED"
+								msg: "RESPONSE.CREATED",
 							}));
 							hasResponseStarted = true;
 						}
@@ -204,12 +293,9 @@ export const connectToElevenLabs = async (
 							user,
 						);
 
-                        // Flush any remaining audio before sending complete
-                        opus.flush(true);
-
-                        // Send response complete with device info like OpenAI does
 						console.log("Sending RESPONSE.COMPLETE to ESP32");
-                        hasResponseStarted = false; // Reset for next response
+						opus.flush(true);
+						hasResponseStarted = false;
 						try {
 							const device = await getDeviceInfo(supabase, user.user_id);
 							ws.send(JSON.stringify({
@@ -228,16 +314,21 @@ export const connectToElevenLabs = async (
 					break;
 
 				case "vad_score":
-                    // Voice Activity Detection score - can be used for debugging
 					if (event.vad_score_event?.vad_score) {
 						console.log("VAD score:", event.vad_score_event.vad_score);
 					}
 					break;
 
 				case "internal_tentative_agent_response":
-                    // Tentative response while agent is thinking
+					if (
-                    if (event.tentative_agent_response_internal_event?.tentative_agent_response) {
+						event.tentative_agent_response_internal_event
-                        console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response);
+							?.tentative_agent_response
+					) {
+						console.log(
+							"Tentative response:",
+							event.tentative_agent_response_internal_event
+								.tentative_agent_response,
+						);
 					}
 					break;
 
@@ -245,7 +336,7 @@ export const connectToElevenLabs = async (
 					console.log("ElevenLabs conversation ended");
 					ws.send(JSON.stringify({
 						type: "server",
-                        msg: "SESSION.END"
+						msg: "SESSION.END",
 					}));
 					break;
 
@@ -256,46 +347,30 @@ export const connectToElevenLabs = async (
 
 		elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
 			console.log("ElevenLabs connection closed:", details.reason);
+			opus.close();
 			ws.close();
 		});
 
-        // Process queued messages
+		// Match OpenAI/Gemini flow by triggering the first assistant turn immediately.
+		if (firstMessage?.trim()) {
+			console.log("Sending initial user_message to ElevenLabs to start first turn");
+			elevenLabsConnection.sendMessage({
+				type: "user_message",
+				text: firstMessage,
+			});
+		}
+
+		// Process queued messages.
 		while (messageQueue.length > 0) {
 			const queuedMessage = messageQueue.shift();
 			if (queuedMessage) {
-                handleClientMessage(queuedMessage, false);
+				handleClientMessage(queuedMessage.data, queuedMessage.isBinary);
 			}
 		}
-
-        // Set up ESP32 WebSocket handlers
-        ws.on("message", (data: any, isBinary: boolean) => {
-            if (!isElevenLabsConnected) {
-                messageQueue.push(data);
-            } else {
-                handleClientMessage(data, isBinary);
-            }
-        });
-
-        ws.on("error", (error: any) => {
-            console.error("ESP32 WebSocket error:", error);
-            elevenLabsConnection?.close();
-        });
-
-        ws.on("close", async (code: number, reason: string) => {
-            console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
-            await closeHandler();
-            opus.close();
-            elevenLabsConnection?.close();
-
-            if (isDev && connectionPcmFile) {
-                connectionPcmFile.close();
-                console.log("Closed debug audio file.");
-            }
-        });
 	} catch (error) {
 		console.error("Failed to connect to ElevenLabs:", error);
+		opus.close();
 
-        // Send more specific error information
 		let errorMessage = "RESPONSE.ERROR";
 		if (error instanceof Error) {
 			console.error("Error details:", error.message);
@@ -306,7 +381,7 @@ export const connectToElevenLabs = async (
 
 		ws.send(JSON.stringify({
 			type: "server",
-            msg: errorMessage
+			msg: errorMessage,
 		}));
 	}
 };