admin-valentin/AIvoices
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixing eleven labs api

Browse source
This commit is contained in:
akdeb 2026-02-17 16:43:33 +07:00
parent 0407b4a561
commit fe5a0c8cce
3 changed files with 343 additions and 281 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
firmware-arduino/src/Config.cpp
View file

@ -33,11 +33,11 @@ volatile bool sleepRequested = false;
*/ */
#ifdef DEV_MODE #ifdef DEV_MODE
const char *ws_server = "192.168.1.155"; const char *ws_server = "192.168.1.121";
const uint16_t ws_port = 8000; const uint16_t ws_port = 8000;
const char *ws_path = "/"; const char *ws_path = "/";
// Backend server details // Backend server details
const char *backend_server = "192.168.1.155"; const char *backend_server = "192.168.1.121";
const uint16_t backend_port = 3000; const uint16_t backend_port = 3000;
#elif defined(PROD_MODE) #elif defined(PROD_MODE)

19
server-deno/main.ts
View file

@ -4,7 +4,7 @@ import type {
WebSocket as WSWebSocket, WebSocket as WSWebSocket,
WebSocketServer as _WebSocketServer, WebSocketServer as _WebSocketServer,
} from "npm:@types/ws"; } from "npm:@types/ws";
import { authenticateUser, elevenLabsApiKey } from "./utils.ts"; import { authenticateUser } from "./utils.ts";
import { import {
createFirstMessage, createFirstMessage,
createSystemPrompt, createSystemPrompt,
@ -59,7 +59,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
ws.send( ws.send(
JSON.stringify({ JSON.stringify({
type: "auth", type: "auth",
volume_control: user.device?.volume ?? 20, volume_control: user.device?.volume ?? 100,
is_ota: user.device?.is_ota ?? false, is_ota: user.device?.is_ota ?? false,
is_reset: user.device?.is_reset ?? false, is_reset: user.device?.is_reset ?? false,
pitch_factor: user.personality?.pitch_factor ?? 1, pitch_factor: user.personality?.pitch_factor ?? 1,
@ -92,20 +92,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
await connectToGrok(providerArgs); await connectToGrok(providerArgs);
break; break;
case "elevenlabs": case "elevenlabs":
const agentId = user.personality?.oai_voice ?? ""; await connectToElevenLabs(providerArgs);
if (!elevenLabsApiKey) {
throw new Error("ELEVENLABS_API_KEY environment variable is required");
}
await connectToElevenLabs(
ws,
payload,
connectionPcmFile,
agentId,
elevenLabsApiKey,
closeHandler,
);
break; break;
case "hume": case "hume":
await connectToHume(providerArgs); await connectToHume(providerArgs);

593
server-deno/models/elevenlabs.ts
View file

@ -2,311 +2,386 @@ import { Buffer } from "node:buffer";
import type { RawData } from "npm:@types/ws"; import type { RawData } from "npm:@types/ws";
// @ts-ignore // @ts-ignore
import { import {
WebSocketConnection, WebSocketConnection,
type SessionConfig, type SessionConfig,
type IncomingSocketEvent, type IncomingSocketEvent,
type DisconnectionDetails type DisconnectionDetails
} from "npm:@elevenlabs/client"; } from "npm:@elevenlabs/client";
import { addConversation, getDeviceInfo } from "../supabase.ts"; import { addConversation, getDeviceInfo } from "../supabase.ts";
import { createOpusPacketizer, isDev } from "../utils.ts"; import {
createOpusPacketizer,
isDev,
elevenLabsApiKey,
SAMPLE_RATE,
} from "../utils.ts";
// Calculate audio level for debugging // Calculate audio level for debugging
function calculateAudioLevel(audioData: any): number { function calculateAudioLevel(audioData: any): number {
if (!audioData || audioData.length === 0) return 0; if (!audioData || audioData.length === 0) return 0;
// Convert to 16-bit samples // Convert to 16-bit samples
const samples = new Int16Array(audioData.buffer || audioData); const samples = new Int16Array(audioData.buffer || audioData);
let sum = 0; let sum = 0;
for (let i = 0; i < samples.length; i++) { for (let i = 0; i < samples.length; i++) {
sum += Math.abs(samples[i]); sum += Math.abs(samples[i]);
} }
return Math.round(sum / samples.length); return Math.round(sum / samples.length);
} }
export const connectToElevenLabs = async ( // Resample mono PCM16 little-endian audio with linear interpolation.
ws: WebSocket, function resamplePcm16Mono(
payload: IPayload, inputBytes: Buffer,
connectionPcmFile: Deno.FsFile | null, fromRate: number,
agentId: string, toRate: number,
apiKey: string, ): Buffer {
closeHandler: () => Promise<void>, if (fromRate === toRate || inputBytes.length === 0) {
) => { return inputBytes;
console.log(apiKey, agentId); }
const { user, supabase } = payload;
const opus = createOpusPacketizer((packet) => ws.send(packet)); const inputSamples = inputBytes.length / 2;
const outputSamples = Math.max(1, Math.floor((inputSamples * toRate) / fromRate));
const output = Buffer.alloc(outputSamples * 2);
// Queue messages until ElevenLabs connection is ready for (let i = 0; i < outputSamples; i++) {
const messageQueue: RawData[] = []; const sourcePos = (i * fromRate) / toRate;
let isElevenLabsConnected = false; const leftIndex = Math.floor(sourcePos);
let elevenLabsConnection: WebSocketConnection | null = null; const rightIndex = Math.min(leftIndex + 1, inputSamples - 1);
let hasResponseStarted = false; const frac = sourcePos - leftIndex;
// Handle messages from ESP32 client const left = inputBytes.readInt16LE(leftIndex * 2);
const handleClientMessage = async (data: any, isBinary: boolean) => { const right = inputBytes.readInt16LE(rightIndex * 2);
try { const sample = Math.round(left + (right - left) * frac);
if (isBinary) { output.writeInt16LE(sample, i * 2);
const base64Data = data.toString("base64"); }
if (isDev && connectionPcmFile) { return output;
await connectionPcmFile.write(data); }
}
// Send audio to ElevenLabs using their client export const connectToElevenLabs = async ({
if (isElevenLabsConnected && elevenLabsConnection) { ws,
// Check if audio contains actual speech (simple volume check) payload,
const audioLevel = calculateAudioLevel(data); connectionPcmFile,
console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`); firstMessage,
closeHandler,
}: ProviderArgs) => {
const agentId = payload.user.personality?.voice?.config?.config_id ??
payload.user.personality?.oai_voice;
const apiKey = elevenLabsApiKey;
try { if (!agentId || !apiKey) {
elevenLabsConnection.sendMessage({ throw new Error("Agent ID or API key is missing");
user_audio_chunk: base64Data, }
});
} catch (error) {
console.error("Error sending audio to ElevenLabs:", error);
}
} else {
console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`);
}
} else {
const message = JSON.parse(data.toString("utf-8"));
if (message.type === "instruction") { const { user, supabase } = payload;
switch (message.msg) { const opus = createOpusPacketizer((packet) => ws.send(packet));
case "INTERRUPT":
console.log("Interrupt detected");
if (elevenLabsConnection) {
elevenLabsConnection.sendMessage({
type: "user_activity"
});
}
break;
case "END_SESSION": // Queue messages until ElevenLabs connection is ready.
console.log("End session requested"); const messageQueue: Array<{ data: RawData; isBinary: boolean }> = [];
if (elevenLabsConnection) { let isElevenLabsConnected = false;
elevenLabsConnection.close(); let elevenLabsConnection: WebSocketConnection | null = null;
} let hasResponseStarted = false;
break; let elevenInputSampleRate = 16000;
} let elevenOutputSampleRate = 16000;
}
}
} catch (error) {
console.error("Error handling client message:", error);
}
};
try { // Handle messages from ESP32 ws client.
// For server-side usage, we need to get a signed URL first const handleClientMessage = async (data: any, isBinary: boolean) => {
const signedUrlResponse = await fetch( try {
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`, if (isBinary) {
{ if (isDev && connectionPcmFile) {
headers: { await connectionPcmFile.write(data);
'xi-api-key': apiKey, }
},
}
);
if (!signedUrlResponse.ok) { // Send audio to ElevenLabs using the expected input sample rate.
throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`); if (isElevenLabsConnected && elevenLabsConnection) {
} const sourceBuffer = Buffer.from(data);
const pcmForEleven = resamplePcm16Mono(
sourceBuffer,
SAMPLE_RATE,
elevenInputSampleRate,
);
const base64Data = pcmForEleven.toString("base64");
const { signed_url } = await signedUrlResponse.json(); const audioLevel = calculateAudioLevel(data);
console.log(
`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, resampled=${pcmForEleven.length} bytes, inRate=${SAMPLE_RATE}, elevenInRate=${elevenInputSampleRate}, level=${audioLevel}`,
);
// Use default audio formats (let ElevenLabs auto-detect) try {
const modifiedSignedUrl = signed_url; elevenLabsConnection.sendMessage({
user_audio_chunk: base64Data,
});
} catch (error) {
console.error("Error sending audio to ElevenLabs:", error);
}
} else {
console.log(
`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`,
);
}
} else {
const message = JSON.parse(data.toString("utf-8"));
// Create ElevenLabs connection using signed URL for server-side usage if (message.type === "instruction") {
const sessionConfig: SessionConfig = { switch (message.msg) {
signedUrl: modifiedSignedUrl, case "INTERRUPT":
connectionType: "websocket", console.log("Interrupt detected");
}; if (elevenLabsConnection) {
elevenLabsConnection.sendMessage({
type: "user_activity",
});
}
break;
elevenLabsConnection = await WebSocketConnection.create(sessionConfig); case "END_SESSION":
console.log("End session requested");
if (elevenLabsConnection) {
elevenLabsConnection.close();
}
break;
}
}
}
} catch (error) {
console.error("Error handling client message:", error);
}
};
console.log("Connected to ElevenLabs successfully!"); // Register handlers immediately so early ESP32 audio is not dropped while ElevenLabs connects.
isElevenLabsConnected = true; ws.on("message", (data: any, isBinary: boolean) => {
console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`); if (!isElevenLabsConnected) {
messageQueue.push({ data, isBinary });
} else {
handleClientMessage(data, isBinary);
}
});
// Set up ElevenLabs event handlers ws.on("error", (error: any) => {
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => { console.error("ESP32 WebSocket error:", error);
console.log("ElevenLabs message type:", event); opus.close();
elevenLabsConnection?.close();
});
switch (event.type) { ws.on("close", async (code: number, reason: string) => {
case "conversation_initiation_metadata": console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
console.log("ElevenLabs conversation initiated (metadata received)"); await closeHandler();
break; opus.close();
elevenLabsConnection?.close();
case "ping": if (isDev && connectionPcmFile) {
// Handle ping messages - send pong response connectionPcmFile.close();
console.log("Received ping from ElevenLabs, sending pong"); console.log("Closed debug audio file.");
if (event.ping_event?.event_id) { }
elevenLabsConnection.sendMessage({ });
type: "pong",
event_id: event.ping_event.event_id
});
}
break;
case "audio": try {
if (event.audio_event?.audio_base_64) { // For server-side usage, we need to get a signed URL first.
// Send RESPONSE.CREATED before first audio chunk const signedUrlResponse = await fetch(
if (!hasResponseStarted) { `https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)"); {
opus.reset(); headers: {
ws.send(JSON.stringify({ "xi-api-key": apiKey,
type: "server", },
msg: "RESPONSE.CREATED" },
})); );
hasResponseStarted = true;
}
const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64"); if (!signedUrlResponse.ok) {
console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes`); throw new Error(
`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`,
);
}
// Use Opus packetizer to encode and send audio const { signed_url } = await signedUrlResponse.json();
opus.push(audioBuffer);
}
break;
case "user_transcript": const sessionConfig: SessionConfig = {
if (event.user_transcription_event?.user_transcript) { signedUrl: signed_url,
console.log("User transcript:", event.user_transcription_event.user_transcript); connectionType: "websocket",
addConversation( };
supabase,
"user",
event.user_transcription_event.user_transcript,
user,
);
if (!hasResponseStarted) { elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)"); elevenInputSampleRate = elevenLabsConnection.inputFormat.sampleRate;
opus.reset(); elevenOutputSampleRate = elevenLabsConnection.outputFormat.sampleRate;
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED"
}));
hasResponseStarted = true;
}
}
break;
case "agent_response": console.log("Connected to ElevenLabs successfully!");
if (event.agent_response_event?.agent_response) { console.log(
console.log("Agent response:", event.agent_response_event.agent_response); `ElevenLabs formats: input=${elevenLabsConnection.inputFormat.format}_${elevenInputSampleRate}, output=${elevenLabsConnection.outputFormat.format}_${elevenOutputSampleRate}`,
addConversation( );
supabase, isElevenLabsConnected = true;
"assistant", console.log(
event.agent_response_event.agent_response, "ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK",
user, );
);
// Flush any remaining audio before sending complete elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
opus.flush(true); console.log("ElevenLabs message type:", event);
// Send response complete with device info like OpenAI does switch (event.type) {
console.log("Sending RESPONSE.COMPLETE to ESP32"); case "conversation_initiation_metadata":
hasResponseStarted = false; // Reset for next response console.log("ElevenLabs conversation initiated (metadata received)");
try { break;
const device = await getDeviceInfo(supabase, user.user_id);
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
volume_control: device?.volume ?? 100,
}));
} catch (error) {
console.error("Error fetching updated device info:", error);
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
}));
}
}
break;
case "vad_score": case "ping":
// Voice Activity Detection score - can be used for debugging console.log("Received ping from ElevenLabs, sending pong");
if (event.vad_score_event?.vad_score) { if (event.ping_event?.event_id) {
console.log("VAD score:", event.vad_score_event.vad_score); elevenLabsConnection?.sendMessage({
} type: "pong",
break; event_id: event.ping_event.event_id,
});
}
break;
case "internal_tentative_agent_response": case "audio":
// Tentative response while agent is thinking if (event.audio_event?.audio_base_64) {
if (event.tentative_agent_response_internal_event?.tentative_agent_response) { if (!hasResponseStarted) {
console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response); console.log(
} "Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
break; );
opus.reset();
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED",
}));
hasResponseStarted = true;
}
case "conversation_end": const audioBuffer = Buffer.from(
console.log("ElevenLabs conversation ended"); event.audio_event.audio_base_64,
ws.send(JSON.stringify({ "base64",
type: "server", );
msg: "SESSION.END" const pcmForEsp32 = resamplePcm16Mono(
})); audioBuffer,
break; elevenOutputSampleRate,
SAMPLE_RATE,
);
opus.push(pcmForEsp32);
}
break;
default: case "user_transcript":
console.log("Unknown ElevenLabs message:", event.type, event); if (event.user_transcription_event?.user_transcript) {
} console.log(
}); "User transcript:",
event.user_transcription_event.user_transcript,
);
addConversation(
supabase,
"user",
event.user_transcription_event.user_transcript,
user,
);
elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => { if (!hasResponseStarted) {
console.log("ElevenLabs connection closed:", details.reason); console.log(
ws.close(); "Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
}); );
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED",
}));
hasResponseStarted = true;
}
}
break;
// Process queued messages case "agent_response":
while (messageQueue.length > 0) { if (event.agent_response_event?.agent_response) {
const queuedMessage = messageQueue.shift(); console.log("Agent response:", event.agent_response_event.agent_response);
if (queuedMessage) { addConversation(
handleClientMessage(queuedMessage, false); supabase,
} "assistant",
} event.agent_response_event.agent_response,
user,
);
// Set up ESP32 WebSocket handlers console.log("Sending RESPONSE.COMPLETE to ESP32");
ws.on("message", (data: any, isBinary: boolean) => { opus.flush(true);
if (!isElevenLabsConnected) { hasResponseStarted = false;
messageQueue.push(data); try {
} else { const device = await getDeviceInfo(supabase, user.user_id);
handleClientMessage(data, isBinary); ws.send(JSON.stringify({
} type: "server",
}); msg: "RESPONSE.COMPLETE",
volume_control: device?.volume ?? 100,
}));
} catch (error) {
console.error("Error fetching updated device info:", error);
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
}));
}
}
break;
ws.on("error", (error: any) => { case "vad_score":
console.error("ESP32 WebSocket error:", error); if (event.vad_score_event?.vad_score) {
elevenLabsConnection?.close(); console.log("VAD score:", event.vad_score_event.vad_score);
}); }
break;
ws.on("close", async (code: number, reason: string) => { case "internal_tentative_agent_response":
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`); if (
await closeHandler(); event.tentative_agent_response_internal_event
opus.close(); ?.tentative_agent_response
elevenLabsConnection?.close(); ) {
console.log(
"Tentative response:",
event.tentative_agent_response_internal_event
.tentative_agent_response,
);
}
break;
if (isDev && connectionPcmFile) { case "conversation_end":
connectionPcmFile.close(); console.log("ElevenLabs conversation ended");
console.log("Closed debug audio file."); ws.send(JSON.stringify({
} type: "server",
}); msg: "SESSION.END",
} catch (error) { }));
console.error("Failed to connect to ElevenLabs:", error); break;
// Send more specific error information default:
let errorMessage = "RESPONSE.ERROR"; console.log("Unknown ElevenLabs message:", event.type, event);
if (error instanceof Error) { }
console.error("Error details:", error.message); });
if (error.message.includes("signed URL")) {
errorMessage = "AUTH.ERROR";
}
}
ws.send(JSON.stringify({ elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
type: "server", console.log("ElevenLabs connection closed:", details.reason);
msg: errorMessage opus.close();
})); ws.close();
} });
// Match OpenAI/Gemini flow by triggering the first assistant turn immediately.
if (firstMessage?.trim()) {
console.log("Sending initial user_message to ElevenLabs to start first turn");
elevenLabsConnection.sendMessage({
type: "user_message",
text: firstMessage,
});
}
// Process queued messages.
while (messageQueue.length > 0) {
const queuedMessage = messageQueue.shift();
if (queuedMessage) {
handleClientMessage(queuedMessage.data, queuedMessage.isBinary);
}
}
} catch (error) {
console.error("Failed to connect to ElevenLabs:", error);
opus.close();
let errorMessage = "RESPONSE.ERROR";
if (error instanceof Error) {
console.error("Error details:", error.message);
if (error.message.includes("signed URL")) {
errorMessage = "AUTH.ERROR";
}
}
ws.send(JSON.stringify({
type: "server",
msg: errorMessage,
}));
}
}; };