348 lines
No EOL
14 KiB
TypeScript
348 lines
No EOL
14 KiB
TypeScript
import { Buffer } from "node:buffer";
|
|
import type { RawData } from "npm:@types/ws";
|
|
// @ts-ignore
|
|
import {
|
|
WebSocketConnection,
|
|
type SessionConfig,
|
|
type IncomingSocketEvent,
|
|
type DisconnectionDetails
|
|
} from "npm:@elevenlabs/client";
|
|
|
|
import { addConversation, getDeviceInfo } from "../supabase.ts";
|
|
import { encoder, FRAME_SIZE, isDev } from "../utils.ts";
|
|
|
|
// Calculate audio level for debugging
|
|
function calculateAudioLevel(audioData: any): number {
|
|
if (!audioData || audioData.length === 0) return 0;
|
|
|
|
// Convert to 16-bit samples
|
|
const samples = new Int16Array(audioData.buffer || audioData);
|
|
let sum = 0;
|
|
|
|
for (let i = 0; i < samples.length; i++) {
|
|
sum += Math.abs(samples[i]);
|
|
}
|
|
|
|
return Math.round(sum / samples.length);
|
|
}
|
|
|
|
export const connectToElevenLabs = async (
|
|
ws: WebSocket,
|
|
payload: IPayload,
|
|
connectionPcmFile: Deno.FsFile | null,
|
|
agentId: string,
|
|
apiKey: string,
|
|
) => {
|
|
console.log(apiKey, agentId);
|
|
const { user, supabase } = payload;
|
|
|
|
// Queue messages until ElevenLabs connection is ready
|
|
const messageQueue: RawData[] = [];
|
|
let isElevenLabsConnected = false;
|
|
let elevenLabsConnection: WebSocketConnection | null = null;
|
|
let hasResponseStarted = false;
|
|
|
|
// Handle messages from ESP32 client
|
|
const handleClientMessage = async (data: any, isBinary: boolean) => {
|
|
try {
|
|
if (isBinary) {
|
|
const base64Data = data.toString("base64");
|
|
|
|
if (isDev && connectionPcmFile) {
|
|
await connectionPcmFile.write(data);
|
|
}
|
|
|
|
// Send audio to ElevenLabs using their client
|
|
if (isElevenLabsConnected && elevenLabsConnection) {
|
|
// Check if audio contains actual speech (simple volume check)
|
|
const audioLevel = calculateAudioLevel(data);
|
|
console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`);
|
|
|
|
try {
|
|
elevenLabsConnection.sendMessage({
|
|
user_audio_chunk: base64Data,
|
|
});
|
|
} catch (error) {
|
|
console.error("Error sending audio to ElevenLabs:", error);
|
|
}
|
|
} else {
|
|
console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`);
|
|
}
|
|
} else {
|
|
const message = JSON.parse(data.toString("utf-8"));
|
|
|
|
if (message.type === "instruction") {
|
|
switch (message.msg) {
|
|
case "INTERRUPT":
|
|
console.log("Interrupt detected");
|
|
if (elevenLabsConnection) {
|
|
elevenLabsConnection.sendMessage({
|
|
type: "user_activity"
|
|
});
|
|
}
|
|
break;
|
|
|
|
case "END_SESSION":
|
|
console.log("End session requested");
|
|
if (elevenLabsConnection) {
|
|
elevenLabsConnection.close();
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error("Error handling client message:", error);
|
|
}
|
|
};
|
|
|
|
try {
|
|
// For server-side usage, we need to get a signed URL first
|
|
const signedUrlResponse = await fetch(
|
|
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
|
|
{
|
|
headers: {
|
|
'xi-api-key': apiKey,
|
|
},
|
|
}
|
|
);
|
|
|
|
if (!signedUrlResponse.ok) {
|
|
throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`);
|
|
}
|
|
|
|
const { signed_url } = await signedUrlResponse.json();
|
|
|
|
// Use default audio formats (let ElevenLabs auto-detect)
|
|
const modifiedSignedUrl = signed_url;
|
|
|
|
// Create ElevenLabs connection using signed URL for server-side usage
|
|
const sessionConfig: SessionConfig = {
|
|
signedUrl: modifiedSignedUrl,
|
|
connectionType: "websocket",
|
|
};
|
|
|
|
elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
|
|
|
|
console.log("Connected to ElevenLabs successfully!");
|
|
isElevenLabsConnected = true;
|
|
console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`);
|
|
|
|
// Send initial RESPONSE.CREATED for the first message
|
|
// console.log("Sending initial RESPONSE.CREATED to ESP32");
|
|
// ws.send(JSON.stringify({
|
|
// type: "server",
|
|
// msg: "RESPONSE.CREATED"
|
|
// }));
|
|
|
|
// Set up ElevenLabs event handlers
|
|
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
|
|
console.log("ElevenLabs message type:", event);
|
|
|
|
switch (event.type) {
|
|
case "conversation_initiation_metadata":
|
|
console.log("ElevenLabs conversation initiated (metadata received)");
|
|
// RESPONSE.CREATED already sent when connection was established
|
|
break;
|
|
|
|
case "ping":
|
|
// Handle ping messages - send pong response
|
|
console.log("Received ping from ElevenLabs, sending pong");
|
|
if (event.ping_event?.event_id) {
|
|
elevenLabsConnection.sendMessage({
|
|
type: "pong",
|
|
event_id: event.ping_event.event_id
|
|
});
|
|
}
|
|
break;
|
|
|
|
case "audio":
|
|
if (event.audio_event?.audio_base_64) {
|
|
// Send RESPONSE.CREATED only for the first audio chunk of each response
|
|
if (!hasResponseStarted) {
|
|
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.CREATED"
|
|
}));
|
|
hasResponseStarted = true;
|
|
}
|
|
|
|
const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64");
|
|
console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes, processing into ${Math.ceil(audioBuffer.length / FRAME_SIZE)} frames`);
|
|
|
|
let framesSent = 0;
|
|
// Process audio in frames for Opus encoding
|
|
for (let offset = 0; offset < audioBuffer.length; offset += FRAME_SIZE) {
|
|
const frame = audioBuffer.subarray(offset, offset + FRAME_SIZE);
|
|
|
|
try {
|
|
const encodedPacket = encoder.encode(frame);
|
|
ws.send(encodedPacket);
|
|
framesSent++;
|
|
} catch (_e) {
|
|
// Skip this frame but continue with others
|
|
console.log(`Failed to encode frame at offset ${offset}`);
|
|
}
|
|
}
|
|
console.log(`Sent ${framesSent} audio frames to ESP32`);
|
|
}
|
|
break;
|
|
|
|
case "user_transcript":
|
|
if (event.user_transcription_event?.user_transcript) {
|
|
console.log("User transcript:", event.user_transcription_event.user_transcript);
|
|
addConversation(
|
|
supabase,
|
|
"user",
|
|
event.user_transcription_event.user_transcript,
|
|
user,
|
|
);
|
|
|
|
// Send audio committed message like OpenAI does
|
|
// console.log("Sending AUDIO.COMMITTED to ESP32");
|
|
// ws.send(JSON.stringify({
|
|
// type: "server",
|
|
// msg: "AUDIO.COMMITTED"
|
|
// }));
|
|
|
|
if (!hasResponseStarted) {
|
|
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.CREATED"
|
|
}));
|
|
hasResponseStarted = true;
|
|
}
|
|
|
|
|
|
}
|
|
break;
|
|
|
|
case "agent_response":
|
|
if (event.agent_response_event?.agent_response) {
|
|
console.log("Agent response:", event.agent_response_event.agent_response);
|
|
addConversation(
|
|
supabase,
|
|
"assistant",
|
|
event.agent_response_event.agent_response,
|
|
user,
|
|
);
|
|
|
|
// Send response complete with device info like OpenAI does
|
|
console.log("Sending RESPONSE.COMPLETE to ESP32");
|
|
hasResponseStarted = false; // Reset for next response
|
|
try {
|
|
const device = await getDeviceInfo(supabase, user.user_id);
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.COMPLETE",
|
|
volume_control: device?.volume ?? 100,
|
|
}));
|
|
} catch (error) {
|
|
console.error("Error fetching updated device info:", error);
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.COMPLETE",
|
|
}));
|
|
}
|
|
}
|
|
break;
|
|
|
|
case "vad_score":
|
|
// Voice Activity Detection score - can be used for debugging
|
|
if (event.vad_score_event?.vad_score) {
|
|
console.log("VAD score:", event.vad_score_event.vad_score);
|
|
}
|
|
break;
|
|
|
|
case "internal_tentative_agent_response":
|
|
// Tentative response while agent is thinking
|
|
if (event.tentative_agent_response_internal_event?.tentative_agent_response) {
|
|
console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response);
|
|
}
|
|
break;
|
|
|
|
case "conversation_end":
|
|
console.log("ElevenLabs conversation ended");
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "SESSION.END"
|
|
}));
|
|
break;
|
|
|
|
default:
|
|
console.log("Unknown ElevenLabs message:", event.type, event);
|
|
}
|
|
});
|
|
|
|
elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
|
|
console.log("ElevenLabs connection closed:", details.reason);
|
|
ws.close();
|
|
});
|
|
|
|
// Process queued messages
|
|
while (messageQueue.length > 0) {
|
|
const queuedMessage = messageQueue.shift();
|
|
if (queuedMessage) {
|
|
handleClientMessage(queuedMessage, false);
|
|
}
|
|
}
|
|
|
|
// Set up ESP32 WebSocket handlers
|
|
ws.on("message", (data: any, isBinary: boolean) => {
|
|
if (!isElevenLabsConnected) {
|
|
messageQueue.push(data);
|
|
} else {
|
|
handleClientMessage(data, isBinary);
|
|
}
|
|
});
|
|
|
|
ws.on("error", (error: any) => {
|
|
console.error("ESP32 WebSocket error:", error);
|
|
elevenLabsConnection?.close();
|
|
});
|
|
|
|
ws.on("close", async (code: number, reason: string) => {
|
|
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
|
|
elevenLabsConnection?.close();
|
|
|
|
if (isDev && connectionPcmFile) {
|
|
connectionPcmFile.close();
|
|
console.log("Closed debug audio file.");
|
|
}
|
|
});
|
|
|
|
// Send initial device info to ESP32
|
|
try {
|
|
const device = await getDeviceInfo(supabase, user.user_id);
|
|
|
|
if (device) {
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "DEVICE.INFO",
|
|
volume_control: device.volume ?? 100,
|
|
}));
|
|
}
|
|
} catch (error) {
|
|
console.error("Error fetching device info:", error);
|
|
}
|
|
|
|
} catch (error) {
|
|
console.error("Failed to connect to ElevenLabs:", error);
|
|
|
|
// Send more specific error information
|
|
let errorMessage = "RESPONSE.ERROR";
|
|
if (error instanceof Error) {
|
|
console.error("Error details:", error.message);
|
|
if (error.message.includes("signed URL")) {
|
|
errorMessage = "AUTH.ERROR";
|
|
}
|
|
}
|
|
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: errorMessage
|
|
}));
|
|
}
|
|
}; |