376 lines
10 KiB
TypeScript
376 lines
10 KiB
TypeScript
import { Buffer } from "node:buffer";
|
|
import type { RawData } from "npm:@types/ws";
|
|
// @ts-ignore
|
|
import {
|
|
WebSocketConnection,
|
|
type SessionConfig,
|
|
type IncomingSocketEvent,
|
|
type DisconnectionDetails
|
|
} from "npm:@elevenlabs/client";
|
|
import { addConversation, getDeviceInfo } from "../supabase.ts";
|
|
import {
|
|
createOpusPacketizer,
|
|
isDev,
|
|
elevenLabsApiKey,
|
|
SAMPLE_RATE,
|
|
} from "../utils.ts";
|
|
|
|
// Calculate audio level for debugging
|
|
function calculateAudioLevel(audioData: any): number {
|
|
if (!audioData || audioData.length === 0) return 0;
|
|
|
|
// Convert to 16-bit samples
|
|
const samples = new Int16Array(audioData.buffer || audioData);
|
|
let sum = 0;
|
|
|
|
for (let i = 0; i < samples.length; i++) {
|
|
sum += Math.abs(samples[i]);
|
|
}
|
|
|
|
return Math.round(sum / samples.length);
|
|
}
|
|
|
|
// Resample mono PCM16 little-endian audio with linear interpolation.
|
|
function resamplePcm16Mono(
|
|
inputBytes: Buffer,
|
|
fromRate: number,
|
|
toRate: number,
|
|
): Buffer {
|
|
if (fromRate === toRate || inputBytes.length === 0) {
|
|
return inputBytes;
|
|
}
|
|
|
|
const inputSamples = inputBytes.length / 2;
|
|
const outputSamples = Math.max(1, Math.floor((inputSamples * toRate) / fromRate));
|
|
const output = Buffer.alloc(outputSamples * 2);
|
|
|
|
for (let i = 0; i < outputSamples; i++) {
|
|
const sourcePos = (i * fromRate) / toRate;
|
|
const leftIndex = Math.floor(sourcePos);
|
|
const rightIndex = Math.min(leftIndex + 1, inputSamples - 1);
|
|
const frac = sourcePos - leftIndex;
|
|
|
|
const left = inputBytes.readInt16LE(leftIndex * 2);
|
|
const right = inputBytes.readInt16LE(rightIndex * 2);
|
|
const sample = Math.round(left + (right - left) * frac);
|
|
output.writeInt16LE(sample, i * 2);
|
|
}
|
|
|
|
return output;
|
|
}
|
|
|
|
export const connectToElevenLabs = async ({
|
|
ws,
|
|
payload,
|
|
connectionPcmFile,
|
|
firstMessage,
|
|
closeHandler,
|
|
}: ProviderArgs) => {
|
|
const agentId = payload.user.personality?.voice?.config?.config_id ??
|
|
payload.user.personality?.oai_voice;
|
|
const apiKey = elevenLabsApiKey;
|
|
|
|
if (!agentId || !apiKey) {
|
|
throw new Error("Agent ID or API key is missing");
|
|
}
|
|
|
|
const { user, supabase } = payload;
|
|
const opus = createOpusPacketizer((packet) => ws.send(packet));
|
|
|
|
// Queue messages until ElevenLabs connection is ready.
|
|
const messageQueue: Array<{ data: RawData; isBinary: boolean }> = [];
|
|
let isElevenLabsConnected = false;
|
|
let elevenLabsConnection: WebSocketConnection | null = null;
|
|
let hasResponseStarted = false;
|
|
let elevenInputSampleRate = 16000;
|
|
let elevenOutputSampleRate = 24000;
|
|
|
|
// Handle messages from ESP32 ws client.
|
|
const handleClientMessage = async (data: any, isBinary: boolean) => {
|
|
try {
|
|
if (isBinary) {
|
|
if (isDev && connectionPcmFile) {
|
|
await connectionPcmFile.write(data);
|
|
}
|
|
|
|
// Send audio to ElevenLabs using the expected input sample rate.
|
|
if (isElevenLabsConnected && elevenLabsConnection) {
|
|
const sourceBuffer = Buffer.from(data);
|
|
const pcmForEleven = resamplePcm16Mono(
|
|
sourceBuffer,
|
|
SAMPLE_RATE,
|
|
elevenInputSampleRate,
|
|
);
|
|
const base64Data = pcmForEleven.toString("base64");
|
|
|
|
const audioLevel = calculateAudioLevel(data);
|
|
console.log(
|
|
`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, resampled=${pcmForEleven.length} bytes, inRate=${SAMPLE_RATE}, elevenInRate=${elevenInputSampleRate}, level=${audioLevel}`,
|
|
);
|
|
|
|
try {
|
|
elevenLabsConnection.sendMessage({
|
|
user_audio_chunk: base64Data,
|
|
});
|
|
} catch (error) {
|
|
console.error("Error sending audio to ElevenLabs:", error);
|
|
}
|
|
} else {
|
|
console.log(
|
|
`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`,
|
|
);
|
|
}
|
|
} else {
|
|
const message = JSON.parse(data.toString("utf-8"));
|
|
|
|
if (message.type === "instruction") {
|
|
switch (message.msg) {
|
|
case "INTERRUPT":
|
|
console.log("Interrupt detected");
|
|
if (elevenLabsConnection) {
|
|
elevenLabsConnection.sendMessage({
|
|
type: "user_activity",
|
|
});
|
|
}
|
|
break;
|
|
|
|
case "END_SESSION":
|
|
console.log("End session requested");
|
|
if (elevenLabsConnection) {
|
|
elevenLabsConnection.close();
|
|
}
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error("Error handling client message:", error);
|
|
}
|
|
};
|
|
|
|
// Register handlers immediately so early ESP32 audio is not dropped while ElevenLabs connects.
|
|
ws.on("message", (data: any, isBinary: boolean) => {
|
|
if (!isElevenLabsConnected) {
|
|
messageQueue.push({ data, isBinary });
|
|
} else {
|
|
handleClientMessage(data, isBinary);
|
|
}
|
|
});
|
|
|
|
ws.on("error", (error: any) => {
|
|
console.error("ESP32 WebSocket error:", error);
|
|
opus.close();
|
|
elevenLabsConnection?.close();
|
|
});
|
|
|
|
ws.on("close", async (code: number, reason: string) => {
|
|
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
|
|
await closeHandler();
|
|
opus.close();
|
|
elevenLabsConnection?.close();
|
|
|
|
if (isDev && connectionPcmFile) {
|
|
connectionPcmFile.close();
|
|
console.log("Closed debug audio file.");
|
|
}
|
|
});
|
|
|
|
try {
|
|
// For server-side usage, we need to get a signed URL first.
|
|
const signedUrlResponse = await fetch(
|
|
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
|
|
{
|
|
headers: {
|
|
"xi-api-key": apiKey,
|
|
},
|
|
},
|
|
);
|
|
|
|
if (!signedUrlResponse.ok) {
|
|
throw new Error(
|
|
`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`,
|
|
);
|
|
}
|
|
|
|
const { signed_url } = await signedUrlResponse.json();
|
|
|
|
const sessionConfig: SessionConfig = {
|
|
signedUrl: signed_url,
|
|
connectionType: "websocket",
|
|
};
|
|
|
|
elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
|
|
elevenInputSampleRate = elevenLabsConnection.inputFormat.sampleRate;
|
|
elevenOutputSampleRate = elevenLabsConnection.outputFormat.sampleRate;
|
|
|
|
console.log("Connected to ElevenLabs successfully!");
|
|
console.log(
|
|
`ElevenLabs formats: input=${elevenLabsConnection.inputFormat.format}_${elevenInputSampleRate}, output=${elevenLabsConnection.outputFormat.format}_${elevenOutputSampleRate}`,
|
|
);
|
|
isElevenLabsConnected = true;
|
|
console.log(
|
|
"ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK",
|
|
);
|
|
|
|
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
|
|
console.log("ElevenLabs message type:", event);
|
|
|
|
switch (event.type) {
|
|
case "conversation_initiation_metadata":
|
|
console.log("ElevenLabs conversation initiated (metadata received)");
|
|
break;
|
|
|
|
case "ping":
|
|
console.log("Received ping from ElevenLabs, sending pong");
|
|
if (event.ping_event?.event_id) {
|
|
elevenLabsConnection?.sendMessage({
|
|
type: "pong",
|
|
event_id: event.ping_event.event_id,
|
|
});
|
|
}
|
|
break;
|
|
|
|
case "audio":
|
|
if (event.audio_event?.audio_base_64) {
|
|
if (!hasResponseStarted) {
|
|
console.log(
|
|
"Sending RESPONSE.CREATED to ESP32 (agent audio starting)",
|
|
);
|
|
opus.reset();
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.CREATED",
|
|
}));
|
|
hasResponseStarted = true;
|
|
}
|
|
|
|
const audioBuffer = Buffer.from(
|
|
event.audio_event.audio_base_64,
|
|
"base64",
|
|
);
|
|
const pcmForEsp32 = resamplePcm16Mono(
|
|
audioBuffer,
|
|
elevenOutputSampleRate,
|
|
SAMPLE_RATE,
|
|
);
|
|
opus.push(pcmForEsp32);
|
|
}
|
|
break;
|
|
|
|
case "user_transcript":
|
|
if (event.user_transcription_event?.user_transcript) {
|
|
console.log(
|
|
"User transcript:",
|
|
event.user_transcription_event.user_transcript,
|
|
);
|
|
addConversation(
|
|
supabase,
|
|
"user",
|
|
event.user_transcription_event.user_transcript,
|
|
user,
|
|
);
|
|
}
|
|
break;
|
|
|
|
case "agent_response":
|
|
if (event.agent_response_event?.agent_response) {
|
|
console.log("Agent response:", event.agent_response_event.agent_response);
|
|
addConversation(
|
|
supabase,
|
|
"assistant",
|
|
event.agent_response_event.agent_response,
|
|
user,
|
|
);
|
|
|
|
console.log("Sending RESPONSE.COMPLETE to ESP32");
|
|
opus.flush(true);
|
|
hasResponseStarted = false;
|
|
try {
|
|
const device = await getDeviceInfo(supabase, user.user_id);
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.COMPLETE",
|
|
volume_control: device?.volume ?? 100,
|
|
}));
|
|
} catch (error) {
|
|
console.error("Error fetching updated device info:", error);
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "RESPONSE.COMPLETE",
|
|
}));
|
|
}
|
|
}
|
|
break;
|
|
|
|
case "vad_score":
|
|
if (event.vad_score_event?.vad_score) {
|
|
console.log("VAD score:", event.vad_score_event.vad_score);
|
|
}
|
|
break;
|
|
|
|
case "internal_tentative_agent_response":
|
|
if (
|
|
event.tentative_agent_response_internal_event
|
|
?.tentative_agent_response
|
|
) {
|
|
console.log(
|
|
"Tentative response:",
|
|
event.tentative_agent_response_internal_event
|
|
.tentative_agent_response,
|
|
);
|
|
}
|
|
break;
|
|
|
|
case "conversation_end":
|
|
console.log("ElevenLabs conversation ended");
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: "SESSION.END",
|
|
}));
|
|
break;
|
|
|
|
default:
|
|
console.log("Unknown ElevenLabs message:", event.type, event);
|
|
}
|
|
});
|
|
|
|
elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
|
|
console.log("ElevenLabs connection closed:", details);
|
|
opus.close();
|
|
ws.close();
|
|
});
|
|
|
|
// Match OpenAI/Gemini flow by triggering the first assistant turn immediately.
|
|
if (firstMessage?.trim()) {
|
|
console.log("Sending initial user_message to ElevenLabs to start first turn");
|
|
elevenLabsConnection.sendMessage({
|
|
type: "user_message",
|
|
text: firstMessage,
|
|
});
|
|
}
|
|
|
|
// Process queued messages.
|
|
while (messageQueue.length > 0) {
|
|
const queuedMessage = messageQueue.shift();
|
|
if (queuedMessage) {
|
|
handleClientMessage(queuedMessage.data, queuedMessage.isBinary);
|
|
}
|
|
}
|
|
} catch (error) {
|
|
console.error("Failed to connect to ElevenLabs:", error);
|
|
opus.close();
|
|
|
|
let errorMessage = "RESPONSE.ERROR";
|
|
if (error instanceof Error) {
|
|
console.error("Error details:", error.message);
|
|
if (error.message.includes("signed URL")) {
|
|
errorMessage = "AUTH.ERROR";
|
|
}
|
|
}
|
|
|
|
ws.send(JSON.stringify({
|
|
type: "server",
|
|
msg: errorMessage,
|
|
}));
|
|
}
|
|
};
|