fixing eleven labs api
This commit is contained in:
akdeb
parent
0407b4a561
commit
fe5a0c8cce
3 changed files with 343 additions and 281 deletions
|
|
@ -33,11 +33,11 @@ volatile bool sleepRequested = false;
|
|||
*/
|
||||
|
||||
#ifdef DEV_MODE
|
||||
const char *ws_server = "192.168.1.155";
|
||||
const char *ws_server = "192.168.1.121";
|
||||
const uint16_t ws_port = 8000;
|
||||
const char *ws_path = "/";
|
||||
// Backend server details
|
||||
const char *backend_server = "192.168.1.155";
|
||||
const char *backend_server = "192.168.1.121";
|
||||
const uint16_t backend_port = 3000;
|
||||
|
||||
#elif defined(PROD_MODE)
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import type {
|
|||
WebSocket as WSWebSocket,
|
||||
WebSocketServer as _WebSocketServer,
|
||||
} from "npm:@types/ws";
|
||||
import { authenticateUser, elevenLabsApiKey } from "./utils.ts";
|
||||
import { authenticateUser } from "./utils.ts";
|
||||
import {
|
||||
createFirstMessage,
|
||||
createSystemPrompt,
|
||||
|
|
@ -59,7 +59,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
|
|||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "auth",
|
||||
volume_control: user.device?.volume ?? 20,
|
||||
volume_control: user.device?.volume ?? 100,
|
||||
is_ota: user.device?.is_ota ?? false,
|
||||
is_reset: user.device?.is_reset ?? false,
|
||||
pitch_factor: user.personality?.pitch_factor ?? 1,
|
||||
|
|
@ -92,20 +92,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
|
|||
await connectToGrok(providerArgs);
|
||||
break;
|
||||
case "elevenlabs":
|
||||
const agentId = user.personality?.oai_voice ?? "";
|
||||
|
||||
if (!elevenLabsApiKey) {
|
||||
throw new Error("ELEVENLABS_API_KEY environment variable is required");
|
||||
}
|
||||
|
||||
await connectToElevenLabs(
|
||||
ws,
|
||||
payload,
|
||||
connectionPcmFile,
|
||||
agentId,
|
||||
elevenLabsApiKey,
|
||||
closeHandler,
|
||||
);
|
||||
await connectToElevenLabs(providerArgs);
|
||||
break;
|
||||
case "hume":
|
||||
await connectToHume(providerArgs);
|
||||
|
|
|
|||
|
|
@ -2,311 +2,386 @@ import { Buffer } from "node:buffer";
|
|||
import type { RawData } from "npm:@types/ws";
|
||||
// @ts-ignore
|
||||
import {
|
||||
WebSocketConnection,
|
||||
type SessionConfig,
|
||||
type IncomingSocketEvent,
|
||||
type DisconnectionDetails
|
||||
WebSocketConnection,
|
||||
type SessionConfig,
|
||||
type IncomingSocketEvent,
|
||||
type DisconnectionDetails
|
||||
} from "npm:@elevenlabs/client";
|
||||
|
||||
import { addConversation, getDeviceInfo } from "../supabase.ts";
|
||||
import { createOpusPacketizer, isDev } from "../utils.ts";
|
||||
import {
|
||||
createOpusPacketizer,
|
||||
isDev,
|
||||
elevenLabsApiKey,
|
||||
SAMPLE_RATE,
|
||||
} from "../utils.ts";
|
||||
|
||||
// Calculate audio level for debugging
|
||||
function calculateAudioLevel(audioData: any): number {
|
||||
if (!audioData || audioData.length === 0) return 0;
|
||||
|
||||
// Convert to 16-bit samples
|
||||
const samples = new Int16Array(audioData.buffer || audioData);
|
||||
let sum = 0;
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
sum += Math.abs(samples[i]);
|
||||
}
|
||||
|
||||
return Math.round(sum / samples.length);
|
||||
if (!audioData || audioData.length === 0) return 0;
|
||||
|
||||
// Convert to 16-bit samples
|
||||
const samples = new Int16Array(audioData.buffer || audioData);
|
||||
let sum = 0;
|
||||
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
sum += Math.abs(samples[i]);
|
||||
}
|
||||
|
||||
return Math.round(sum / samples.length);
|
||||
}
|
||||
|
||||
export const connectToElevenLabs = async (
|
||||
ws: WebSocket,
|
||||
payload: IPayload,
|
||||
connectionPcmFile: Deno.FsFile | null,
|
||||
agentId: string,
|
||||
apiKey: string,
|
||||
closeHandler: () => Promise<void>,
|
||||
) => {
|
||||
console.log(apiKey, agentId);
|
||||
const { user, supabase } = payload;
|
||||
// Resample mono PCM16 little-endian audio with linear interpolation.
|
||||
function resamplePcm16Mono(
|
||||
inputBytes: Buffer,
|
||||
fromRate: number,
|
||||
toRate: number,
|
||||
): Buffer {
|
||||
if (fromRate === toRate || inputBytes.length === 0) {
|
||||
return inputBytes;
|
||||
}
|
||||
|
||||
const opus = createOpusPacketizer((packet) => ws.send(packet));
|
||||
const inputSamples = inputBytes.length / 2;
|
||||
const outputSamples = Math.max(1, Math.floor((inputSamples * toRate) / fromRate));
|
||||
const output = Buffer.alloc(outputSamples * 2);
|
||||
|
||||
// Queue messages until ElevenLabs connection is ready
|
||||
const messageQueue: RawData[] = [];
|
||||
let isElevenLabsConnected = false;
|
||||
let elevenLabsConnection: WebSocketConnection | null = null;
|
||||
let hasResponseStarted = false;
|
||||
for (let i = 0; i < outputSamples; i++) {
|
||||
const sourcePos = (i * fromRate) / toRate;
|
||||
const leftIndex = Math.floor(sourcePos);
|
||||
const rightIndex = Math.min(leftIndex + 1, inputSamples - 1);
|
||||
const frac = sourcePos - leftIndex;
|
||||
|
||||
// Handle messages from ESP32 client
|
||||
const handleClientMessage = async (data: any, isBinary: boolean) => {
|
||||
try {
|
||||
if (isBinary) {
|
||||
const base64Data = data.toString("base64");
|
||||
const left = inputBytes.readInt16LE(leftIndex * 2);
|
||||
const right = inputBytes.readInt16LE(rightIndex * 2);
|
||||
const sample = Math.round(left + (right - left) * frac);
|
||||
output.writeInt16LE(sample, i * 2);
|
||||
}
|
||||
|
||||
if (isDev && connectionPcmFile) {
|
||||
await connectionPcmFile.write(data);
|
||||
}
|
||||
return output;
|
||||
}
|
||||
|
||||
// Send audio to ElevenLabs using their client
|
||||
if (isElevenLabsConnected && elevenLabsConnection) {
|
||||
// Check if audio contains actual speech (simple volume check)
|
||||
const audioLevel = calculateAudioLevel(data);
|
||||
console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`);
|
||||
|
||||
try {
|
||||
elevenLabsConnection.sendMessage({
|
||||
user_audio_chunk: base64Data,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error sending audio to ElevenLabs:", error);
|
||||
}
|
||||
} else {
|
||||
console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`);
|
||||
}
|
||||
} else {
|
||||
const message = JSON.parse(data.toString("utf-8"));
|
||||
export const connectToElevenLabs = async ({
|
||||
ws,
|
||||
payload,
|
||||
connectionPcmFile,
|
||||
firstMessage,
|
||||
closeHandler,
|
||||
}: ProviderArgs) => {
|
||||
const agentId = payload.user.personality?.voice?.config?.config_id ??
|
||||
payload.user.personality?.oai_voice;
|
||||
const apiKey = elevenLabsApiKey;
|
||||
|
||||
if (message.type === "instruction") {
|
||||
switch (message.msg) {
|
||||
case "INTERRUPT":
|
||||
console.log("Interrupt detected");
|
||||
if (elevenLabsConnection) {
|
||||
elevenLabsConnection.sendMessage({
|
||||
type: "user_activity"
|
||||
});
|
||||
}
|
||||
break;
|
||||
if (!agentId || !apiKey) {
|
||||
throw new Error("Agent ID or API key is missing");
|
||||
}
|
||||
|
||||
case "END_SESSION":
|
||||
console.log("End session requested");
|
||||
if (elevenLabsConnection) {
|
||||
elevenLabsConnection.close();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error handling client message:", error);
|
||||
}
|
||||
};
|
||||
const { user, supabase } = payload;
|
||||
const opus = createOpusPacketizer((packet) => ws.send(packet));
|
||||
|
||||
try {
|
||||
// For server-side usage, we need to get a signed URL first
|
||||
const signedUrlResponse = await fetch(
|
||||
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
|
||||
{
|
||||
headers: {
|
||||
'xi-api-key': apiKey,
|
||||
},
|
||||
}
|
||||
);
|
||||
// Queue messages until ElevenLabs connection is ready.
|
||||
const messageQueue: Array<{ data: RawData; isBinary: boolean }> = [];
|
||||
let isElevenLabsConnected = false;
|
||||
let elevenLabsConnection: WebSocketConnection | null = null;
|
||||
let hasResponseStarted = false;
|
||||
let elevenInputSampleRate = 16000;
|
||||
let elevenOutputSampleRate = 16000;
|
||||
|
||||
if (!signedUrlResponse.ok) {
|
||||
throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`);
|
||||
}
|
||||
// Handle messages from ESP32 ws client.
|
||||
const handleClientMessage = async (data: any, isBinary: boolean) => {
|
||||
try {
|
||||
if (isBinary) {
|
||||
if (isDev && connectionPcmFile) {
|
||||
await connectionPcmFile.write(data);
|
||||
}
|
||||
|
||||
const { signed_url } = await signedUrlResponse.json();
|
||||
// Send audio to ElevenLabs using the expected input sample rate.
|
||||
if (isElevenLabsConnected && elevenLabsConnection) {
|
||||
const sourceBuffer = Buffer.from(data);
|
||||
const pcmForEleven = resamplePcm16Mono(
|
||||
sourceBuffer,
|
||||
SAMPLE_RATE,
|
||||
elevenInputSampleRate,
|
||||
);
|
||||
const base64Data = pcmForEleven.toString("base64");
|
||||
|
||||
// Use default audio formats (let ElevenLabs auto-detect)
|
||||
const modifiedSignedUrl = signed_url;
|
||||
const audioLevel = calculateAudioLevel(data);
|
||||
console.log(
|
||||
`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, resampled=${pcmForEleven.length} bytes, inRate=${SAMPLE_RATE}, elevenInRate=${elevenInputSampleRate}, level=${audioLevel}`,
|
||||
);
|
||||
|
||||
// Create ElevenLabs connection using signed URL for server-side usage
|
||||
const sessionConfig: SessionConfig = {
|
||||
signedUrl: modifiedSignedUrl,
|
||||
connectionType: "websocket",
|
||||
};
|
||||
try {
|
||||
elevenLabsConnection.sendMessage({
|
||||
user_audio_chunk: base64Data,
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Error sending audio to ElevenLabs:", error);
|
||||
}
|
||||
} else {
|
||||
console.log(
|
||||
`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`,
|
||||
);
|
||||
}
|
||||
} else {
|
||||
const message = JSON.parse(data.toString("utf-8"));
|
||||
|
||||
elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
|
||||
if (message.type === "instruction") {
|
||||
switch (message.msg) {
|
||||
case "INTERRUPT":
|
||||
console.log("Interrupt detected");
|
||||
if (elevenLabsConnection) {
|
||||
elevenLabsConnection.sendMessage({
|
||||
type: "user_activity",
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
console.log("Connected to ElevenLabs successfully!");
|
||||
isElevenLabsConnected = true;
|
||||
console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`);
|
||||
case "END_SESSION":
|
||||
console.log("End session requested");
|
||||
if (elevenLabsConnection) {
|
||||
elevenLabsConnection.close();
|
||||
}
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Error handling client message:", error);
|
||||
}
|
||||
};
|
||||
|
||||
// Set up ElevenLabs event handlers
|
||||
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
|
||||
console.log("ElevenLabs message type:", event);
|
||||
// Register handlers immediately so early ESP32 audio is not dropped while ElevenLabs connects.
|
||||
ws.on("message", (data: any, isBinary: boolean) => {
|
||||
if (!isElevenLabsConnected) {
|
||||
messageQueue.push({ data, isBinary });
|
||||
} else {
|
||||
handleClientMessage(data, isBinary);
|
||||
}
|
||||
});
|
||||
|
||||
switch (event.type) {
|
||||
case "conversation_initiation_metadata":
|
||||
console.log("ElevenLabs conversation initiated (metadata received)");
|
||||
break;
|
||||
ws.on("error", (error: any) => {
|
||||
console.error("ESP32 WebSocket error:", error);
|
||||
opus.close();
|
||||
elevenLabsConnection?.close();
|
||||
});
|
||||
|
||||
case "ping":
|
||||
// Handle ping messages - send pong response
|
||||
console.log("Received ping from ElevenLabs, sending pong");
|
||||
if (event.ping_event?.event_id) {
|
||||
elevenLabsConnection.sendMessage({
|
||||
type: "pong",
|
||||
event_id: event.ping_event.event_id
|
||||
});
|
||||
}
|
||||
break;
|
||||
ws.on("close", async (code: number, reason: string) => {
|
||||
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
|
||||
await closeHandler();
|
||||
opus.close();
|
||||
elevenLabsConnection?.close();
|
||||
|
||||
case "audio":
|
||||
if (event.audio_event?.audio_base_64) {
|
||||
// Send RESPONSE.CREATED before first audio chunk
|
||||
if (!hasResponseStarted) {
|
||||
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
|
||||
opus.reset();
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.CREATED"
|
||||
}));
|
||||
hasResponseStarted = true;
|
||||
}
|
||||
if (isDev && connectionPcmFile) {
|
||||
connectionPcmFile.close();
|
||||
console.log("Closed debug audio file.");
|
||||
}
|
||||
});
|
||||
|
||||
const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64");
|
||||
console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes`);
|
||||
try {
|
||||
// For server-side usage, we need to get a signed URL first.
|
||||
const signedUrlResponse = await fetch(
|
||||
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
|
||||
{
|
||||
headers: {
|
||||
"xi-api-key": apiKey,
|
||||
},
|
||||
},
|
||||
);
|
||||
|
||||
// Use Opus packetizer to encode and send audio
|
||||
opus.push(audioBuffer);
|
||||
}
|
||||
break;
|
||||
if (!signedUrlResponse.ok) {
|
||||
throw new Error(
|
||||
`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
case "user_transcript":
|
||||
if (event.user_transcription_event?.user_transcript) {
|
||||
console.log("User transcript:", event.user_transcription_event.user_transcript);
|
||||
addConversation(
|
||||
supabase,
|
||||
"user",
|
||||
event.user_transcription_event.user_transcript,
|
||||
user,
|
||||
);
|
||||
const { signed_url } = await signedUrlResponse.json();
|
||||
|
||||
if (!hasResponseStarted) {
|
||||
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
|
||||
opus.reset();
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.CREATED"
|
||||
}));
|
||||
hasResponseStarted = true;
|
||||
}
|
||||
}
|
||||
break;
|
||||
const sessionConfig: SessionConfig = {
|
||||
signedUrl: signed_url,
|
||||
connectionType: "websocket",
|
||||
};
|
||||
|
||||
case "agent_response":
|
||||
if (event.agent_response_event?.agent_response) {
|
||||
console.log("Agent response:", event.agent_response_event.agent_response);
|
||||
addConversation(
|
||||
supabase,
|
||||
"assistant",
|
||||
event.agent_response_event.agent_response,
|
||||
user,
|
||||
);
|
||||
elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
|
||||
elevenInputSampleRate = elevenLabsConnection.inputFormat.sampleRate;
|
||||
elevenOutputSampleRate = elevenLabsConnection.outputFormat.sampleRate;
|
||||
|
||||
// Flush any remaining audio before sending complete
|
||||
opus.flush(true);
|
||||
console.log("Connected to ElevenLabs successfully!");
|
||||
console.log(
|
||||
`ElevenLabs formats: input=${elevenLabsConnection.inputFormat.format}_${elevenInputSampleRate}, output=${elevenLabsConnection.outputFormat.format}_${elevenOutputSampleRate}`,
|
||||
);
|
||||
isElevenLabsConnected = true;
|
||||
console.log(
|
||||
"ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK",
|
||||
);
|
||||
|
||||
// Send response complete with device info like OpenAI does
|
||||
console.log("Sending RESPONSE.COMPLETE to ESP32");
|
||||
hasResponseStarted = false; // Reset for next response
|
||||
try {
|
||||
const device = await getDeviceInfo(supabase, user.user_id);
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.COMPLETE",
|
||||
volume_control: device?.volume ?? 100,
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error("Error fetching updated device info:", error);
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.COMPLETE",
|
||||
}));
|
||||
}
|
||||
}
|
||||
break;
|
||||
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
|
||||
console.log("ElevenLabs message type:", event);
|
||||
|
||||
case "vad_score":
|
||||
// Voice Activity Detection score - can be used for debugging
|
||||
if (event.vad_score_event?.vad_score) {
|
||||
console.log("VAD score:", event.vad_score_event.vad_score);
|
||||
}
|
||||
break;
|
||||
switch (event.type) {
|
||||
case "conversation_initiation_metadata":
|
||||
console.log("ElevenLabs conversation initiated (metadata received)");
|
||||
break;
|
||||
|
||||
case "internal_tentative_agent_response":
|
||||
// Tentative response while agent is thinking
|
||||
if (event.tentative_agent_response_internal_event?.tentative_agent_response) {
|
||||
console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response);
|
||||
}
|
||||
break;
|
||||
case "ping":
|
||||
console.log("Received ping from ElevenLabs, sending pong");
|
||||
if (event.ping_event?.event_id) {
|
||||
elevenLabsConnection?.sendMessage({
|
||||
type: "pong",
|
||||
event_id: event.ping_event.event_id,
|
||||
});
|
||||
}
|
||||
break;
|
||||
|
||||
case "conversation_end":
|
||||
console.log("ElevenLabs conversation ended");
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "SESSION.END"
|
||||
}));
|
||||
break;
|
||||
case "audio":
|
||||
if (event.audio_event?.audio_base_64) {
|
||||
if (!hasResponseStarted) {
|
||||
console.log(
|
||||
"Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
|
||||
);
|
||||
opus.reset();
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.CREATED",
|
||||
}));
|
||||
hasResponseStarted = true;
|
||||
}
|
||||
|
||||
default:
|
||||
console.log("Unknown ElevenLabs message:", event.type, event);
|
||||
}
|
||||
});
|
||||
const audioBuffer = Buffer.from(
|
||||
event.audio_event.audio_base_64,
|
||||
"base64",
|
||||
);
|
||||
const pcmForEsp32 = resamplePcm16Mono(
|
||||
audioBuffer,
|
||||
elevenOutputSampleRate,
|
||||
SAMPLE_RATE,
|
||||
);
|
||||
opus.push(pcmForEsp32);
|
||||
}
|
||||
break;
|
||||
|
||||
elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
|
||||
console.log("ElevenLabs connection closed:", details.reason);
|
||||
ws.close();
|
||||
});
|
||||
case "user_transcript":
|
||||
if (event.user_transcription_event?.user_transcript) {
|
||||
console.log(
|
||||
"User transcript:",
|
||||
event.user_transcription_event.user_transcript,
|
||||
);
|
||||
addConversation(
|
||||
supabase,
|
||||
"user",
|
||||
event.user_transcription_event.user_transcript,
|
||||
user,
|
||||
);
|
||||
|
||||
// Process queued messages
|
||||
while (messageQueue.length > 0) {
|
||||
const queuedMessage = messageQueue.shift();
|
||||
if (queuedMessage) {
|
||||
handleClientMessage(queuedMessage, false);
|
||||
}
|
||||
}
|
||||
if (!hasResponseStarted) {
|
||||
console.log(
|
||||
"Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
|
||||
);
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.CREATED",
|
||||
}));
|
||||
hasResponseStarted = true;
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
// Set up ESP32 WebSocket handlers
|
||||
ws.on("message", (data: any, isBinary: boolean) => {
|
||||
if (!isElevenLabsConnected) {
|
||||
messageQueue.push(data);
|
||||
} else {
|
||||
handleClientMessage(data, isBinary);
|
||||
}
|
||||
});
|
||||
case "agent_response":
|
||||
if (event.agent_response_event?.agent_response) {
|
||||
console.log("Agent response:", event.agent_response_event.agent_response);
|
||||
addConversation(
|
||||
supabase,
|
||||
"assistant",
|
||||
event.agent_response_event.agent_response,
|
||||
user,
|
||||
);
|
||||
|
||||
ws.on("error", (error: any) => {
|
||||
console.error("ESP32 WebSocket error:", error);
|
||||
elevenLabsConnection?.close();
|
||||
});
|
||||
console.log("Sending RESPONSE.COMPLETE to ESP32");
|
||||
opus.flush(true);
|
||||
hasResponseStarted = false;
|
||||
try {
|
||||
const device = await getDeviceInfo(supabase, user.user_id);
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.COMPLETE",
|
||||
volume_control: device?.volume ?? 100,
|
||||
}));
|
||||
} catch (error) {
|
||||
console.error("Error fetching updated device info:", error);
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "RESPONSE.COMPLETE",
|
||||
}));
|
||||
}
|
||||
}
|
||||
break;
|
||||
|
||||
ws.on("close", async (code: number, reason: string) => {
|
||||
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
|
||||
await closeHandler();
|
||||
opus.close();
|
||||
elevenLabsConnection?.close();
|
||||
case "vad_score":
|
||||
if (event.vad_score_event?.vad_score) {
|
||||
console.log("VAD score:", event.vad_score_event.vad_score);
|
||||
}
|
||||
break;
|
||||
|
||||
if (isDev && connectionPcmFile) {
|
||||
connectionPcmFile.close();
|
||||
console.log("Closed debug audio file.");
|
||||
}
|
||||
});
|
||||
} catch (error) {
|
||||
console.error("Failed to connect to ElevenLabs:", error);
|
||||
case "internal_tentative_agent_response":
|
||||
if (
|
||||
event.tentative_agent_response_internal_event
|
||||
?.tentative_agent_response
|
||||
) {
|
||||
console.log(
|
||||
"Tentative response:",
|
||||
event.tentative_agent_response_internal_event
|
||||
.tentative_agent_response,
|
||||
);
|
||||
}
|
||||
break;
|
||||
|
||||
// Send more specific error information
|
||||
let errorMessage = "RESPONSE.ERROR";
|
||||
if (error instanceof Error) {
|
||||
console.error("Error details:", error.message);
|
||||
if (error.message.includes("signed URL")) {
|
||||
errorMessage = "AUTH.ERROR";
|
||||
}
|
||||
}
|
||||
case "conversation_end":
|
||||
console.log("ElevenLabs conversation ended");
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: "SESSION.END",
|
||||
}));
|
||||
break;
|
||||
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: errorMessage
|
||||
}));
|
||||
}
|
||||
default:
|
||||
console.log("Unknown ElevenLabs message:", event.type, event);
|
||||
}
|
||||
});
|
||||
|
||||
elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
|
||||
console.log("ElevenLabs connection closed:", details.reason);
|
||||
opus.close();
|
||||
ws.close();
|
||||
});
|
||||
|
||||
// Match OpenAI/Gemini flow by triggering the first assistant turn immediately.
|
||||
if (firstMessage?.trim()) {
|
||||
console.log("Sending initial user_message to ElevenLabs to start first turn");
|
||||
elevenLabsConnection.sendMessage({
|
||||
type: "user_message",
|
||||
text: firstMessage,
|
||||
});
|
||||
}
|
||||
|
||||
// Process queued messages.
|
||||
while (messageQueue.length > 0) {
|
||||
const queuedMessage = messageQueue.shift();
|
||||
if (queuedMessage) {
|
||||
handleClientMessage(queuedMessage.data, queuedMessage.isBinary);
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.error("Failed to connect to ElevenLabs:", error);
|
||||
opus.close();
|
||||
|
||||
let errorMessage = "RESPONSE.ERROR";
|
||||
if (error instanceof Error) {
|
||||
console.error("Error details:", error.message);
|
||||
if (error.message.includes("signed URL")) {
|
||||
errorMessage = "AUTH.ERROR";
|
||||
}
|
||||
}
|
||||
|
||||
ws.send(JSON.stringify({
|
||||
type: "server",
|
||||
msg: errorMessage,
|
||||
}));
|
||||
}
|
||||
};
|
||||
|
|
|
|||
Loading…
Reference in a new issue