From fe5a0c8cce3d5729a0e7b0132d21e87a8994e100 Mon Sep 17 00:00:00 2001 From: akdeb Date: Tue, 17 Feb 2026 16:43:33 +0700 Subject: [PATCH] fixing eleven labs api --- firmware-arduino/src/Config.cpp | 4 +- server-deno/main.ts | 19 +- server-deno/models/elevenlabs.ts | 601 +++++++++++++++++-------------- 3 files changed, 343 insertions(+), 281 deletions(-) diff --git a/firmware-arduino/src/Config.cpp b/firmware-arduino/src/Config.cpp index 135fb08..8ef0607 100644 --- a/firmware-arduino/src/Config.cpp +++ b/firmware-arduino/src/Config.cpp @@ -33,11 +33,11 @@ volatile bool sleepRequested = false; */ #ifdef DEV_MODE -const char *ws_server = "192.168.1.155"; +const char *ws_server = "192.168.1.121"; const uint16_t ws_port = 8000; const char *ws_path = "/"; // Backend server details -const char *backend_server = "192.168.1.155"; +const char *backend_server = "192.168.1.121"; const uint16_t backend_port = 3000; #elif defined(PROD_MODE) diff --git a/server-deno/main.ts b/server-deno/main.ts index c2b6313..5642f34 100644 --- a/server-deno/main.ts +++ b/server-deno/main.ts @@ -4,7 +4,7 @@ import type { WebSocket as WSWebSocket, WebSocketServer as _WebSocketServer, } from "npm:@types/ws"; -import { authenticateUser, elevenLabsApiKey } from "./utils.ts"; +import { authenticateUser } from "./utils.ts"; import { createFirstMessage, createSystemPrompt, @@ -59,7 +59,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { ws.send( JSON.stringify({ type: "auth", - volume_control: user.device?.volume ?? 20, + volume_control: user.device?.volume ?? 100, is_ota: user.device?.is_ota ?? false, is_reset: user.device?.is_reset ?? false, pitch_factor: user.personality?.pitch_factor ?? 1, @@ -92,20 +92,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { await connectToGrok(providerArgs); break; case "elevenlabs": - const agentId = user.personality?.oai_voice ?? ""; - - if (!elevenLabsApiKey) { - throw new Error("ELEVENLABS_API_KEY environment variable is required"); - } - - await connectToElevenLabs( - ws, - payload, - connectionPcmFile, - agentId, - elevenLabsApiKey, - closeHandler, - ); + await connectToElevenLabs(providerArgs); break; case "hume": await connectToHume(providerArgs); diff --git a/server-deno/models/elevenlabs.ts b/server-deno/models/elevenlabs.ts index 85ae3ee..550c838 100644 --- a/server-deno/models/elevenlabs.ts +++ b/server-deno/models/elevenlabs.ts @@ -2,311 +2,386 @@ import { Buffer } from "node:buffer"; import type { RawData } from "npm:@types/ws"; // @ts-ignore import { - WebSocketConnection, - type SessionConfig, - type IncomingSocketEvent, - type DisconnectionDetails + WebSocketConnection, + type SessionConfig, + type IncomingSocketEvent, + type DisconnectionDetails } from "npm:@elevenlabs/client"; - import { addConversation, getDeviceInfo } from "../supabase.ts"; -import { createOpusPacketizer, isDev } from "../utils.ts"; +import { + createOpusPacketizer, + isDev, + elevenLabsApiKey, + SAMPLE_RATE, +} from "../utils.ts"; // Calculate audio level for debugging function calculateAudioLevel(audioData: any): number { - if (!audioData || audioData.length === 0) return 0; - - // Convert to 16-bit samples - const samples = new Int16Array(audioData.buffer || audioData); - let sum = 0; - - for (let i = 0; i < samples.length; i++) { - sum += Math.abs(samples[i]); - } - - return Math.round(sum / samples.length); + if (!audioData || audioData.length === 0) return 0; + + // Convert to 16-bit samples + const samples = new Int16Array(audioData.buffer || audioData); + let sum = 0; + + for (let i = 0; i < samples.length; i++) { + sum += Math.abs(samples[i]); + } + + return Math.round(sum / samples.length); } -export const connectToElevenLabs = async ( - ws: WebSocket, - payload: IPayload, - connectionPcmFile: Deno.FsFile | null, - agentId: string, - apiKey: string, - closeHandler: () => Promise, -) => { - console.log(apiKey, agentId); - const { user, supabase } = payload; +// Resample mono PCM16 little-endian audio with linear interpolation. +function resamplePcm16Mono( + inputBytes: Buffer, + fromRate: number, + toRate: number, +): Buffer { + if (fromRate === toRate || inputBytes.length === 0) { + return inputBytes; + } - const opus = createOpusPacketizer((packet) => ws.send(packet)); + const inputSamples = inputBytes.length / 2; + const outputSamples = Math.max(1, Math.floor((inputSamples * toRate) / fromRate)); + const output = Buffer.alloc(outputSamples * 2); - // Queue messages until ElevenLabs connection is ready - const messageQueue: RawData[] = []; - let isElevenLabsConnected = false; - let elevenLabsConnection: WebSocketConnection | null = null; - let hasResponseStarted = false; + for (let i = 0; i < outputSamples; i++) { + const sourcePos = (i * fromRate) / toRate; + const leftIndex = Math.floor(sourcePos); + const rightIndex = Math.min(leftIndex + 1, inputSamples - 1); + const frac = sourcePos - leftIndex; - // Handle messages from ESP32 client - const handleClientMessage = async (data: any, isBinary: boolean) => { - try { - if (isBinary) { - const base64Data = data.toString("base64"); + const left = inputBytes.readInt16LE(leftIndex * 2); + const right = inputBytes.readInt16LE(rightIndex * 2); + const sample = Math.round(left + (right - left) * frac); + output.writeInt16LE(sample, i * 2); + } - if (isDev && connectionPcmFile) { - await connectionPcmFile.write(data); - } + return output; +} - // Send audio to ElevenLabs using their client - if (isElevenLabsConnected && elevenLabsConnection) { - // Check if audio contains actual speech (simple volume check) - const audioLevel = calculateAudioLevel(data); - console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`); - - try { - elevenLabsConnection.sendMessage({ - user_audio_chunk: base64Data, - }); - } catch (error) { - console.error("Error sending audio to ElevenLabs:", error); - } - } else { - console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`); - } - } else { - const message = JSON.parse(data.toString("utf-8")); +export const connectToElevenLabs = async ({ + ws, + payload, + connectionPcmFile, + firstMessage, + closeHandler, +}: ProviderArgs) => { + const agentId = payload.user.personality?.voice?.config?.config_id ?? + payload.user.personality?.oai_voice; + const apiKey = elevenLabsApiKey; - if (message.type === "instruction") { - switch (message.msg) { - case "INTERRUPT": - console.log("Interrupt detected"); - if (elevenLabsConnection) { - elevenLabsConnection.sendMessage({ - type: "user_activity" - }); - } - break; + if (!agentId || !apiKey) { + throw new Error("Agent ID or API key is missing"); + } - case "END_SESSION": - console.log("End session requested"); - if (elevenLabsConnection) { - elevenLabsConnection.close(); - } - break; - } - } - } - } catch (error) { - console.error("Error handling client message:", error); - } - }; + const { user, supabase } = payload; + const opus = createOpusPacketizer((packet) => ws.send(packet)); - try { - // For server-side usage, we need to get a signed URL first - const signedUrlResponse = await fetch( - `https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`, - { - headers: { - 'xi-api-key': apiKey, - }, - } - ); + // Queue messages until ElevenLabs connection is ready. + const messageQueue: Array<{ data: RawData; isBinary: boolean }> = []; + let isElevenLabsConnected = false; + let elevenLabsConnection: WebSocketConnection | null = null; + let hasResponseStarted = false; + let elevenInputSampleRate = 16000; + let elevenOutputSampleRate = 16000; - if (!signedUrlResponse.ok) { - throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`); - } + // Handle messages from ESP32 ws client. + const handleClientMessage = async (data: any, isBinary: boolean) => { + try { + if (isBinary) { + if (isDev && connectionPcmFile) { + await connectionPcmFile.write(data); + } - const { signed_url } = await signedUrlResponse.json(); + // Send audio to ElevenLabs using the expected input sample rate. + if (isElevenLabsConnected && elevenLabsConnection) { + const sourceBuffer = Buffer.from(data); + const pcmForEleven = resamplePcm16Mono( + sourceBuffer, + SAMPLE_RATE, + elevenInputSampleRate, + ); + const base64Data = pcmForEleven.toString("base64"); - // Use default audio formats (let ElevenLabs auto-detect) - const modifiedSignedUrl = signed_url; + const audioLevel = calculateAudioLevel(data); + console.log( + `Sending audio chunk to ElevenLabs: raw=${data.length} bytes, resampled=${pcmForEleven.length} bytes, inRate=${SAMPLE_RATE}, elevenInRate=${elevenInputSampleRate}, level=${audioLevel}`, + ); - // Create ElevenLabs connection using signed URL for server-side usage - const sessionConfig: SessionConfig = { - signedUrl: modifiedSignedUrl, - connectionType: "websocket", - }; + try { + elevenLabsConnection.sendMessage({ + user_audio_chunk: base64Data, + }); + } catch (error) { + console.error("Error sending audio to ElevenLabs:", error); + } + } else { + console.log( + `Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`, + ); + } + } else { + const message = JSON.parse(data.toString("utf-8")); - elevenLabsConnection = await WebSocketConnection.create(sessionConfig); + if (message.type === "instruction") { + switch (message.msg) { + case "INTERRUPT": + console.log("Interrupt detected"); + if (elevenLabsConnection) { + elevenLabsConnection.sendMessage({ + type: "user_activity", + }); + } + break; - console.log("Connected to ElevenLabs successfully!"); - isElevenLabsConnected = true; - console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`); + case "END_SESSION": + console.log("End session requested"); + if (elevenLabsConnection) { + elevenLabsConnection.close(); + } + break; + } + } + } + } catch (error) { + console.error("Error handling client message:", error); + } + }; - // Set up ElevenLabs event handlers - elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => { - console.log("ElevenLabs message type:", event); + // Register handlers immediately so early ESP32 audio is not dropped while ElevenLabs connects. + ws.on("message", (data: any, isBinary: boolean) => { + if (!isElevenLabsConnected) { + messageQueue.push({ data, isBinary }); + } else { + handleClientMessage(data, isBinary); + } + }); - switch (event.type) { - case "conversation_initiation_metadata": - console.log("ElevenLabs conversation initiated (metadata received)"); - break; + ws.on("error", (error: any) => { + console.error("ESP32 WebSocket error:", error); + opus.close(); + elevenLabsConnection?.close(); + }); - case "ping": - // Handle ping messages - send pong response - console.log("Received ping from ElevenLabs, sending pong"); - if (event.ping_event?.event_id) { - elevenLabsConnection.sendMessage({ - type: "pong", - event_id: event.ping_event.event_id - }); - } - break; + ws.on("close", async (code: number, reason: string) => { + console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`); + await closeHandler(); + opus.close(); + elevenLabsConnection?.close(); - case "audio": - if (event.audio_event?.audio_base_64) { - // Send RESPONSE.CREATED before first audio chunk - if (!hasResponseStarted) { - console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)"); - opus.reset(); - ws.send(JSON.stringify({ - type: "server", - msg: "RESPONSE.CREATED" - })); - hasResponseStarted = true; - } + if (isDev && connectionPcmFile) { + connectionPcmFile.close(); + console.log("Closed debug audio file."); + } + }); - const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64"); - console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes`); + try { + // For server-side usage, we need to get a signed URL first. + const signedUrlResponse = await fetch( + `https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`, + { + headers: { + "xi-api-key": apiKey, + }, + }, + ); - // Use Opus packetizer to encode and send audio - opus.push(audioBuffer); - } - break; + if (!signedUrlResponse.ok) { + throw new Error( + `Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`, + ); + } - case "user_transcript": - if (event.user_transcription_event?.user_transcript) { - console.log("User transcript:", event.user_transcription_event.user_transcript); - addConversation( - supabase, - "user", - event.user_transcription_event.user_transcript, - user, - ); + const { signed_url } = await signedUrlResponse.json(); - if (!hasResponseStarted) { - console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)"); - opus.reset(); - ws.send(JSON.stringify({ - type: "server", - msg: "RESPONSE.CREATED" - })); - hasResponseStarted = true; - } - } - break; + const sessionConfig: SessionConfig = { + signedUrl: signed_url, + connectionType: "websocket", + }; - case "agent_response": - if (event.agent_response_event?.agent_response) { - console.log("Agent response:", event.agent_response_event.agent_response); - addConversation( - supabase, - "assistant", - event.agent_response_event.agent_response, - user, - ); + elevenLabsConnection = await WebSocketConnection.create(sessionConfig); + elevenInputSampleRate = elevenLabsConnection.inputFormat.sampleRate; + elevenOutputSampleRate = elevenLabsConnection.outputFormat.sampleRate; - // Flush any remaining audio before sending complete - opus.flush(true); + console.log("Connected to ElevenLabs successfully!"); + console.log( + `ElevenLabs formats: input=${elevenLabsConnection.inputFormat.format}_${elevenInputSampleRate}, output=${elevenLabsConnection.outputFormat.format}_${elevenOutputSampleRate}`, + ); + isElevenLabsConnected = true; + console.log( + "ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK", + ); - // Send response complete with device info like OpenAI does - console.log("Sending RESPONSE.COMPLETE to ESP32"); - hasResponseStarted = false; // Reset for next response - try { - const device = await getDeviceInfo(supabase, user.user_id); - ws.send(JSON.stringify({ - type: "server", - msg: "RESPONSE.COMPLETE", - volume_control: device?.volume ?? 100, - })); - } catch (error) { - console.error("Error fetching updated device info:", error); - ws.send(JSON.stringify({ - type: "server", - msg: "RESPONSE.COMPLETE", - })); - } - } - break; + elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => { + console.log("ElevenLabs message type:", event); - case "vad_score": - // Voice Activity Detection score - can be used for debugging - if (event.vad_score_event?.vad_score) { - console.log("VAD score:", event.vad_score_event.vad_score); - } - break; + switch (event.type) { + case "conversation_initiation_metadata": + console.log("ElevenLabs conversation initiated (metadata received)"); + break; - case "internal_tentative_agent_response": - // Tentative response while agent is thinking - if (event.tentative_agent_response_internal_event?.tentative_agent_response) { - console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response); - } - break; + case "ping": + console.log("Received ping from ElevenLabs, sending pong"); + if (event.ping_event?.event_id) { + elevenLabsConnection?.sendMessage({ + type: "pong", + event_id: event.ping_event.event_id, + }); + } + break; - case "conversation_end": - console.log("ElevenLabs conversation ended"); - ws.send(JSON.stringify({ - type: "server", - msg: "SESSION.END" - })); - break; + case "audio": + if (event.audio_event?.audio_base_64) { + if (!hasResponseStarted) { + console.log( + "Sending RESPONSE.CREATED to ESP32 (agent audio starting)", + ); + opus.reset(); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.CREATED", + })); + hasResponseStarted = true; + } - default: - console.log("Unknown ElevenLabs message:", event.type, event); - } - }); + const audioBuffer = Buffer.from( + event.audio_event.audio_base_64, + "base64", + ); + const pcmForEsp32 = resamplePcm16Mono( + audioBuffer, + elevenOutputSampleRate, + SAMPLE_RATE, + ); + opus.push(pcmForEsp32); + } + break; - elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => { - console.log("ElevenLabs connection closed:", details.reason); - ws.close(); - }); + case "user_transcript": + if (event.user_transcription_event?.user_transcript) { + console.log( + "User transcript:", + event.user_transcription_event.user_transcript, + ); + addConversation( + supabase, + "user", + event.user_transcription_event.user_transcript, + user, + ); - // Process queued messages - while (messageQueue.length > 0) { - const queuedMessage = messageQueue.shift(); - if (queuedMessage) { - handleClientMessage(queuedMessage, false); - } - } + if (!hasResponseStarted) { + console.log( + "Sending RESPONSE.CREATED to ESP32 (agent audio starting)", + ); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.CREATED", + })); + hasResponseStarted = true; + } + } + break; - // Set up ESP32 WebSocket handlers - ws.on("message", (data: any, isBinary: boolean) => { - if (!isElevenLabsConnected) { - messageQueue.push(data); - } else { - handleClientMessage(data, isBinary); - } - }); + case "agent_response": + if (event.agent_response_event?.agent_response) { + console.log("Agent response:", event.agent_response_event.agent_response); + addConversation( + supabase, + "assistant", + event.agent_response_event.agent_response, + user, + ); - ws.on("error", (error: any) => { - console.error("ESP32 WebSocket error:", error); - elevenLabsConnection?.close(); - }); + console.log("Sending RESPONSE.COMPLETE to ESP32"); + opus.flush(true); + hasResponseStarted = false; + try { + const device = await getDeviceInfo(supabase, user.user_id); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.COMPLETE", + volume_control: device?.volume ?? 100, + })); + } catch (error) { + console.error("Error fetching updated device info:", error); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.COMPLETE", + })); + } + } + break; - ws.on("close", async (code: number, reason: string) => { - console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`); - await closeHandler(); - opus.close(); - elevenLabsConnection?.close(); + case "vad_score": + if (event.vad_score_event?.vad_score) { + console.log("VAD score:", event.vad_score_event.vad_score); + } + break; - if (isDev && connectionPcmFile) { - connectionPcmFile.close(); - console.log("Closed debug audio file."); - } - }); - } catch (error) { - console.error("Failed to connect to ElevenLabs:", error); + case "internal_tentative_agent_response": + if ( + event.tentative_agent_response_internal_event + ?.tentative_agent_response + ) { + console.log( + "Tentative response:", + event.tentative_agent_response_internal_event + .tentative_agent_response, + ); + } + break; - // Send more specific error information - let errorMessage = "RESPONSE.ERROR"; - if (error instanceof Error) { - console.error("Error details:", error.message); - if (error.message.includes("signed URL")) { - errorMessage = "AUTH.ERROR"; - } - } + case "conversation_end": + console.log("ElevenLabs conversation ended"); + ws.send(JSON.stringify({ + type: "server", + msg: "SESSION.END", + })); + break; - ws.send(JSON.stringify({ - type: "server", - msg: errorMessage - })); - } + default: + console.log("Unknown ElevenLabs message:", event.type, event); + } + }); + + elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => { + console.log("ElevenLabs connection closed:", details.reason); + opus.close(); + ws.close(); + }); + + // Match OpenAI/Gemini flow by triggering the first assistant turn immediately. + if (firstMessage?.trim()) { + console.log("Sending initial user_message to ElevenLabs to start first turn"); + elevenLabsConnection.sendMessage({ + type: "user_message", + text: firstMessage, + }); + } + + // Process queued messages. + while (messageQueue.length > 0) { + const queuedMessage = messageQueue.shift(); + if (queuedMessage) { + handleClientMessage(queuedMessage.data, queuedMessage.isBinary); + } + } + } catch (error) { + console.error("Failed to connect to ElevenLabs:", error); + opus.close(); + + let errorMessage = "RESPONSE.ERROR"; + if (error instanceof Error) { + console.error("Error details:", error.message); + if (error.message.includes("signed URL")) { + errorMessage = "AUTH.ERROR"; + } + } + + ws.send(JSON.stringify({ + type: "server", + msg: errorMessage, + })); + } };