admin-valentin/AIvoices
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

opus permits large chunks

Browse source
This commit is contained in:
akdeb 2025-12-26 19:51:15 +07:00
parent 31b1ad6d58
commit f1ed5653d5
8 changed files with 1013 additions and 1029 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

45
server-deno/main.ts
View file

@ -59,40 +59,37 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
ws.send(
JSON.stringify({
type: "auth",
volume_control: user.device?.volume ?? 100,
volume_control: user.device?.volume ?? 20,
is_ota: user.device?.is_ota ?? false,
is_reset: user.device?.is_reset ?? false,
pitch_factor: user.personality?.pitch_factor ?? 1,
}),
);
// Common close handler for cleanup
const closeHandler = async () => {
// Add any common cleanup logic here
};
// Common provider args
const providerArgs: ProviderArgs = {
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
closeHandler,
};
switch (provider) {
case "openai":
await connectToOpenAI(
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
);
await connectToOpenAI(providerArgs);
break;
case "gemini":
await connectToGemini(
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
);
await connectToGemini(providerArgs);
break;
case "grok":
await connectToGrok(
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
);
await connectToGrok(providerArgs);
break;
case "elevenlabs":
const agentId = user.personality?.oai_voice ?? "";
@ -107,11 +104,11 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
connectionPcmFile,
agentId,
elevenLabsApiKey,
closeHandler,
);
break;
case "hume":
await connectToHume(ws, payload,
connectionPcmFile, firstMessage, systemPrompt, () => Promise.resolve());
await connectToHume(providerArgs);
break;
default:
throw new Error(`Unknown provider: ${provider}`);

50
server-deno/models/elevenlabs.ts
View file

@ -9,7 +9,7 @@ import {
} from "npm:@elevenlabs/client";
import { addConversation, getDeviceInfo } from "../supabase.ts";
import { encoder, FRAME_SIZE, isDev } from "../utils.ts";
import { createOpusPacketizer, isDev } from "../utils.ts";
// Calculate audio level for debugging
function calculateAudioLevel(audioData: any): number {
@ -32,10 +32,13 @@ export const connectToElevenLabs = async (
connectionPcmFile: Deno.FsFile | null,
agentId: string,
apiKey: string,
closeHandler: () => Promise<void>,
) => {
console.log(apiKey, agentId);
const { user, supabase } = payload;
const opus = createOpusPacketizer((packet) => ws.send(packet));
// Queue messages until ElevenLabs connection is ready
const messageQueue: RawData[] = [];
let isElevenLabsConnected = false;
@ -128,13 +131,6 @@ export const connectToElevenLabs = async (
isElevenLabsConnected = true;
console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`);
// Send initial RESPONSE.CREATED for the first message
// console.log("Sending initial RESPONSE.CREATED to ESP32");
// ws.send(JSON.stringify({
// type: "server",
// msg: "RESPONSE.CREATED"
// }));
// Set up ElevenLabs event handlers
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
console.log("ElevenLabs message type:", event);
@ -142,7 +138,6 @@ export const connectToElevenLabs = async (
switch (event.type) {
case "conversation_initiation_metadata":
console.log("ElevenLabs conversation initiated (metadata received)");
// RESPONSE.CREATED already sent when connection was established
break;
case "ping":
@ -158,9 +153,10 @@ export const connectToElevenLabs = async (
case "audio":
if (event.audio_event?.audio_base_64) {
// Send RESPONSE.CREATED only for the first audio chunk of each response
// Send RESPONSE.CREATED before first audio chunk
if (!hasResponseStarted) {
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
opus.reset();
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED"
@ -169,23 +165,10 @@ export const connectToElevenLabs = async (
}
const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64");
console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes, processing into ${Math.ceil(audioBuffer.length / FRAME_SIZE)} frames`);
console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes`);
let framesSent = 0;
// Process audio in frames for Opus encoding
for (let offset = 0; offset < audioBuffer.length; offset += FRAME_SIZE) {
const frame = audioBuffer.subarray(offset, offset + FRAME_SIZE);
try {
const encodedPacket = encoder.encode(frame);
ws.send(encodedPacket);
framesSent++;
} catch (_e) {
// Skip this frame but continue with others
console.log(`Failed to encode frame at offset ${offset}`);
}
}
console.log(`Sent ${framesSent} audio frames to ESP32`);
// Use Opus packetizer to encode and send audio
opus.push(audioBuffer);
}
break;
@ -199,23 +182,15 @@ export const connectToElevenLabs = async (
user,
);
// Send audio committed message like OpenAI does
// console.log("Sending AUDIO.COMMITTED to ESP32");
// ws.send(JSON.stringify({
// type: "server",
// msg: "AUDIO.COMMITTED"
// }));
if (!hasResponseStarted) {
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
opus.reset();
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED"
}));
hasResponseStarted = true;
}
}
break;
@ -229,6 +204,9 @@ export const connectToElevenLabs = async (
user,
);
// Flush any remaining audio before sending complete
opus.flush(true);
// Send response complete with device info like OpenAI does
console.log("Sending RESPONSE.COMPLETE to ESP32");
hasResponseStarted = false; // Reset for next response
@ -305,6 +283,8 @@ export const connectToElevenLabs = async (
ws.on("close", async (code: number, reason: string) => {
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
await closeHandler();
opus.close();
elevenLabsConnection?.close();
if (isDev && connectionPcmFile) {

78
server-deno/models/gemini.ts
View file

@ -8,20 +8,23 @@ import {
Modality,
Session,
} from "npm:@google/genai";
import { encoder, FRAME_SIZE, geminiApiKey, isDev } from "../utils.ts";
import { createOpusPacketizer, geminiApiKey, isDev, defaultGeminiVoice } from "../utils.ts";
import { addConversation } from "../supabase.ts";
export const connectToGemini = async (
ws: WebSocket,
payload: IPayload,
connectionPcmFile: Deno.FsFile | null,
firstMessage: string,
systemPrompt: string,
) => {
export const connectToGemini = async ({
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
closeHandler,
}: ProviderArgs) => {
const { user, supabase } = payload;
const { oai_voice } = user.personality ?? { oai_voice: "Sadachbia" };
const voiceName = user.personality?.oai_voice ?? defaultGeminiVoice;
console.log(`Connecting with Gemini key "${geminiApiKey.slice(0, 3)}..."`);
const opus = createOpusPacketizer((packet) => ws.send(packet));
console.log(`Connecting with Gemini key "${geminiApiKey?.slice(0, 3)}..."`);
// Initialize Google GenAI
const ai = new GoogleGenAI({ apiKey: geminiApiKey });
@ -32,15 +35,15 @@ export const connectToGemini = async (
speechConfig: {
voiceConfig: {
prebuiltVoiceConfig: {
voiceName: oai_voice,
voiceName: voiceName,
},
},
},
realtimeInputConfig: {
automaticActivityDetection: {
disabled: false, // Keep VAD enabled
endOfSpeechSensitivity: EndSensitivity.END_SENSITIVITY_LOW, // How sensitive to detect speech ending
silenceDurationMs: 100, // How much silence before considering speech ended
endOfSpeechSensitivity: EndSensitivity.END_SENSITIVITY_LOW,
silenceDurationMs: 100,
},
},
outputAudioTranscription: {},
@ -72,25 +75,15 @@ export const connectToGemini = async (
const message = await waitMessage();
turns.push(message);
if (
message.serverContent
) {
if (message.serverContent) {
if (message.serverContent.generationComplete) {
opus.reset();
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED",
}));
done = true;
}
// if (message.serverContent.turnComplete) {
// ws.send(
// JSON.stringify({
// type: "server",
// msg: "AUDIO.COMMITTED",
// }),
// );
// }
}
}
return turns;
@ -125,35 +118,12 @@ export const connectToGemini = async (
const audioBuffer = new Int16Array(combinedAudio);
const buffer = Buffer.from(audioBuffer.buffer);
// PREVIEW AUDIO
// const wf = new WaveFile();
// wf.fromScratch(1, SAMPLE_RATE, "16", audioBuffer);
// const filename = `gemini_response_${Date.now()}.wav`;
// await Deno.writeFile(filename, wf.toBuffer());
// console.log(`Audio saved as ${filename}`);
// SEND TO ESP32
// Send audio in chunks to client
for (
let offset = 0;
offset < buffer.length;
offset += FRAME_SIZE
) {
const frame = buffer.subarray(
offset,
offset + FRAME_SIZE,
);
try {
const encodedPacket = encoder.encode(frame);
ws.send(encodedPacket);
} catch (_e) {
// Skip this frame but continue with others
}
}
// Use Opus packetizer to encode and send audio
opus.push(buffer);
opus.flush(true);
}
// // Handle text responses if any
// Handle text responses if any
let outputTranscriptionText = "";
let inputTranscriptionText = "";
for (const turn of turns as LiveServerMessage[]) {
@ -256,7 +226,7 @@ export const connectToGemini = async (
geminiSession?.sendRealtimeInput({
audio: {
data: base64Data,
mimeType: "audio/pcm;rate=24000", // Gemini expects 16kHz but 24kHz is fine
mimeType: "audio/pcm;rate=24000",
},
});
}
@ -272,6 +242,8 @@ export const connectToGemini = async (
ws.on("close", async (code: number, reason: string) => {
console.log(`WebSocket closed with code ${code}, reason: ${reason}`);
await closeHandler();
opus.close();
geminiSession?.close();
if (isDev && connectionPcmFile) {
connectionPcmFile.close();

168
server-deno/models/grok.ts
View file

@ -1,32 +1,33 @@
import { Buffer } from "node:buffer";
import type { RawData } from "npm:@types/ws";
import { WebSocket } from "npm:ws";
import { addConversation, getDeviceInfo } from "../supabase.ts";
import { createOpusPacketizer, isDev, xaiApiKey, defaultGrokVoice } from "../utils.ts";
import { Buffer } from 'node:buffer';
import type { RawData } from 'npm:@types/ws';
import { WebSocket } from 'npm:ws';
import { addConversation, getDeviceInfo } from '../supabase.ts';
import { encoder, FRAME_SIZE, isDev, xaiApiKey } from '../utils.ts';
const XAI_REALTIME_URL = "wss://api.x.ai/v1/realtime";
const XAI_REALTIME_URL = 'wss://api.x.ai/v1/realtime';
const DEFAULT_GROK_VOICE = 'Ara';
export const connectToGrok = async (
ws: WebSocket,
payload: IPayload,
connectionPcmFile: Deno.FsFile | null,
firstMessage: string,
systemPrompt: string,
) => {
export const connectToGrok = async ({
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
closeHandler,
}: ProviderArgs) => {
const { user, supabase } = payload;
if (!xaiApiKey) {
throw new Error('XAI_API_KEY is not set');
throw new Error("XAI_API_KEY is not set");
}
const voice = user.personality?.oai_voice ?? DEFAULT_GROK_VOICE;
const voice = user.personality?.oai_voice ?? defaultGrokVoice;
const opus = createOpusPacketizer((packet) => ws.send(packet));
const grokWs = new WebSocket(XAI_REALTIME_URL, {
headers: {
Authorization: `Bearer ${xaiApiKey}`,
'Content-Type': 'application/json',
"Content-Type": "application/json",
},
});
@ -34,21 +35,21 @@ export const connectToGrok = async (
const messageQueue: RawData[] = [];
let createdSent = false;
let outputTranscript = '';
let audioRemainder = Buffer.alloc(0);
let outputTranscript = "";
const sendResponseCreated = async () => {
try {
const device = await getDeviceInfo(supabase, user.user_id);
opus.reset();
ws.send(
JSON.stringify({
type: 'server',
msg: 'RESPONSE.CREATED',
type: "server",
msg: "RESPONSE.CREATED",
volume_control: device?.volume ?? 100,
}),
);
} catch {
ws.send(JSON.stringify({ type: 'server', msg: 'RESPONSE.CREATED' }));
ws.send(JSON.stringify({ type: "server", msg: "RESPONSE.CREATED" }));
}
};
@ -56,30 +57,30 @@ export const connectToGrok = async (
if (!firstMessage) return;
grokWs.send(
JSON.stringify({
type: 'conversation.item.create',
type: "conversation.item.create",
item: {
type: 'message',
role: 'user',
content: [{ type: 'input_text', text: firstMessage }],
type: "message",
role: "user",
content: [{ type: "input_text", text: firstMessage }],
},
}),
);
grokWs.send(JSON.stringify({ type: 'response.create' }));
grokWs.send(JSON.stringify({ type: "response.create" }));
};
grokWs.on('open', () => {
grokWs.on("open", () => {
isConnected = true;
grokWs.send(
JSON.stringify({
type: 'session.update',
type: "session.update",
session: {
voice,
instructions: systemPrompt,
turn_detection: { type: "server_vad" },
audio: {
input: { format: { type: 'audio/pcm', rate: 16000 } },
output: { format: { type: 'audio/pcm', rate: 24000 } },
input: { format: { type: "audio/pcm", rate: 16000 } },
output: { format: { type: "audio/pcm", rate: 24000 } },
},
},
}),
@ -95,91 +96,84 @@ export const connectToGrok = async (
}
});
grokWs.on('message', async (data: Buffer) => {
grokWs.on("message", async (data: Buffer) => {
let event: any;
try {
event = JSON.parse(data.toString('utf-8'));
event = JSON.parse(data.toString("utf-8"));
} catch {
return;
}
try {
switch (event.type) {
case 'response.created':
case "response.created":
if (!createdSent) {
await sendResponseCreated();
createdSent = true;
}
break;
case 'response.output_audio_transcript.delta':
if (typeof event.delta === 'string') {
case "response.output_audio_transcript.delta":
if (typeof event.delta === "string") {
outputTranscript += event.delta;
}
break;
case 'response.output_audio.delta':
if (typeof event.delta === 'string') {
const pcmChunk = Buffer.from(event.delta, 'base64');
audioRemainder = Buffer.concat([audioRemainder, pcmChunk]);
while (audioRemainder.length >= FRAME_SIZE) {
const frame = audioRemainder.subarray(0, FRAME_SIZE);
audioRemainder = audioRemainder.subarray(FRAME_SIZE);
try {
const packet = encoder.encode(frame);
ws.send(packet);
} catch {
// Skip frame
}
}
case "response.output_audio.delta":
if (typeof event.delta === "string") {
const pcmChunk = Buffer.from(event.delta, "base64");
// Use Opus packetizer to encode and send audio
opus.push(pcmChunk);
}
break;
case 'conversation.item.input_audio_transcription.completed':
if (typeof event.transcript === 'string' && event.transcript.length > 0) {
await addConversation(supabase, 'user', event.transcript, user);
case "conversation.item.input_audio_transcription.completed":
if (typeof event.transcript === "string" && event.transcript.length > 0) {
await addConversation(supabase, "user", event.transcript, user);
}
break;
case 'input_audio_buffer.committed':
ws.send(JSON.stringify({ type: 'server', msg: 'AUDIO.COMMITTED' }));
case "input_audio_buffer.committed":
ws.send(JSON.stringify({ type: "server", msg: "AUDIO.COMMITTED" }));
break;
case 'response.done':
case "response.done":
// Flush any remaining audio
opus.flush(true);
if (outputTranscript) {
await addConversation(supabase, 'assistant', outputTranscript, user);
outputTranscript = '';
await addConversation(supabase, "assistant", outputTranscript, user);
outputTranscript = "";
}
ws.send(JSON.stringify({ type: 'server', msg: 'RESPONSE.COMPLETE' }));
ws.send(JSON.stringify({ type: "server", msg: "RESPONSE.COMPLETE" }));
createdSent = false;
break;
case 'error':
ws.send(JSON.stringify({ type: 'server', msg: 'RESPONSE.ERROR' }));
case "error":
ws.send(JSON.stringify({ type: "server", msg: "RESPONSE.ERROR" }));
createdSent = false;
break;
}
} catch (err) {
console.error('Error processing Grok event:', err);
ws.send(JSON.stringify({ type: 'server', msg: 'RESPONSE.ERROR' }));
console.error("Error processing Grok event:", err);
ws.send(JSON.stringify({ type: "server", msg: "RESPONSE.ERROR" }));
createdSent = false;
}
});
grokWs.on('close', () => {
grokWs.on("close", () => {
ws.close();
});
grokWs.on('error', (error: any) => {
console.error('Grok WebSocket error:', error);
ws.send(JSON.stringify({ type: 'server', msg: 'RESPONSE.ERROR' }));
grokWs.on("error", (error: any) => {
console.error("Grok WebSocket error:", error);
ws.send(JSON.stringify({ type: "server", msg: "RESPONSE.ERROR" }));
});
const messageHandler = async (data: RawData, isBinary: boolean) => {
if (isBinary) {
const base64Data = (data as Buffer).toString('base64');
grokWs.send(JSON.stringify({ type: 'input_audio_buffer.append', audio: base64Data }));
const base64Data = (data as Buffer).toString("base64");
grokWs.send(JSON.stringify({ type: "input_audio_buffer.append", audio: base64Data }));
if (isDev && connectionPcmFile) {
await connectionPcmFile.write(data as Buffer);
@ -189,23 +183,23 @@ export const connectToGrok = async (
let message: any;
try {
message = JSON.parse((data as Buffer).toString('utf-8'));
message = JSON.parse((data as Buffer).toString("utf-8"));
} catch {
return;
}
if (message?.type !== 'instruction') return;
if (message?.type !== "instruction") return;
if (message.msg === 'end_of_speech') {
grokWs.send(JSON.stringify({ type: 'input_audio_buffer.commit' }));
grokWs.send(JSON.stringify({ type: 'response.create' }));
grokWs.send(JSON.stringify({ type: 'input_audio_buffer.clear' }));
} else if (message.msg === 'INTERRUPT') {
grokWs.send(JSON.stringify({ type: 'input_audio_buffer.clear' }));
if (message.msg === "end_of_speech") {
grokWs.send(JSON.stringify({ type: "input_audio_buffer.commit" }));
grokWs.send(JSON.stringify({ type: "response.create" }));
grokWs.send(JSON.stringify({ type: "input_audio_buffer.clear" }));
} else if (message.msg === "INTERRUPT") {
grokWs.send(JSON.stringify({ type: "input_audio_buffer.clear" }));
}
};
ws.on('message', (data: RawData, isBinary: boolean) => {
ws.on("message", (data: RawData, isBinary: boolean) => {
if (!isConnected) {
messageQueue.push(data);
} else {
@ -213,13 +207,15 @@ export const connectToGrok = async (
}
});
ws.on('error', (error: any) => {
console.error('ESP32 WebSocket error:', error);
ws.on("error", (error: any) => {
console.error("ESP32 WebSocket error:", error);
grokWs.close();
});
ws.on('close', async (code: number, reason: string) => {
ws.on("close", async (code: number, reason: string) => {
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
await closeHandler();
opus.close();
grokWs.close();
if (isDev && connectionPcmFile) {
connectionPcmFile.close();
@ -227,12 +223,12 @@ export const connectToGrok = async (
});
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => reject(new Error('Grok connection timeout')), 10000);
grokWs.on('open', () => {
const timeout = setTimeout(() => reject(new Error("Grok connection timeout")), 10000);
grokWs.on("open", () => {
clearTimeout(timeout);
resolve();
});
grokWs.on('error', (error: any) => {
grokWs.on("error", (error: any) => {
clearTimeout(timeout);
reject(error);
});

185
server-deno/models/hume.ts
View file

@ -1,31 +1,34 @@
import { Buffer } from 'node:buffer';
import type { RawData } from 'npm:@types/ws';
import { WebSocket } from 'npm:ws';
import { addConversation, getDeviceInfo } from '../supabase.ts';
import { encoder, FRAME_SIZE, isDev, humeApiKey, downsamplePcm, extractPcmFromWav, boostLimitPCM16LEInPlace } from '../utils.ts';
import { Buffer } from "node:buffer";
import type { RawData } from "npm:@types/ws";
import { WebSocket } from "npm:ws";
import { addConversation, getDeviceInfo } from "../supabase.ts";
import { createOpusPacketizer, isDev, humeApiKey, downsamplePcm, extractPcmFromWav, boostLimitPCM16LEInPlace } from "../utils.ts";
export const connectToHume = (
ws: WebSocket,
payload: IPayload,
connectionPcmFile:Deno.FsFile | null,
firstMessage: string,
systemPrompt: string,
closeHandler: () => Promise<void>,
) => {
export const connectToHume = ({
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
closeHandler,
}: ProviderArgs) => {
const { user, supabase } = payload;
const { personality } = user;
const opus = createOpusPacketizer((packet) => ws.send(packet));
console.log(`Connecting to Hume with key "${humeApiKey?.slice(0, 3)}..."`);
// Build Hume WebSocket URL
const configId = personality?.oai_voice ?? "";
const queryParams = new URLSearchParams({
api_key: humeApiKey!,
config_id: personality!.oai_voice,
config_id: configId,
});
const humeWsUrl = `wss://api.hume.ai/v0/evi/chat?${queryParams.toString()}`;
console.log(`Connecting to Hume WebSocket at: ${humeWsUrl.replace(humeApiKey!, 'API_KEY_HIDDEN')}`);
console.log(`Connecting to Hume WebSocket at: ${humeWsUrl.replace(humeApiKey!, "API_KEY_HIDDEN")}`);
const humeWs = new WebSocket(humeWsUrl);
let isConnected = false;
@ -33,14 +36,13 @@ export const connectToHume = (
let createdSent = false;
// Handle Hume WebSocket connection
humeWs.on('open', () => {
console.log('✅ Connected to Hume WebSocket API successfully');
humeWs.on("open", () => {
console.log("✅ Connected to Hume WebSocket API successfully");
isConnected = true;
// Configure Hume session settings for input audio format
// This tells Hume what format we're sending TO them, not what we want back
humeWs.send(JSON.stringify({
type: 'session_settings',
type: "session_settings",
audio: {
encoding: "linear16",
channels: 1,
@ -51,7 +53,7 @@ export const connectToHume = (
// Send simple first message if provided
humeWs.send(JSON.stringify({
type: 'user_input',
type: "user_input",
text: firstMessage,
}));
@ -65,63 +67,64 @@ export const connectToHume = (
});
// Handle messages from Hume
humeWs.on('message', async (data: Buffer) => {
humeWs.on("message", async (data: Buffer) => {
try {
const message: HumeMessage = JSON.parse(data.toString());
console.log(`Received from Hume: ${message.type}`);
switch (message.type) {
case 'assistant_end':
case "assistant_end":
// Flush any remaining audio
opus.flush(true);
// Send RESPONSE.COMPLETE when assistant message is done
ws.send(JSON.stringify({
type: 'server',
msg: 'RESPONSE.COMPLETE',
type: "server",
msg: "RESPONSE.COMPLETE",
}));
// Reset for next turn
createdSent = false;
break;
case 'assistant_message':
case "assistant_message":
const assistantMsg = message as HumeAssistantMessage;
// Store conversation in database
await addConversation(
supabase,
'assistant',
"assistant",
assistantMsg.message.content,
user,
);
break;
case 'audio_output':
case "audio_output":
const audioMsg = message as HumeAudioOutput;
// Send RESPONSE.CREATED before first audio chunk
if (!createdSent) {
try {
const device = await getDeviceInfo(supabase, user.user_id);
opus.reset();
if (device) {
ws.send(JSON.stringify({
type: 'server',
msg: 'RESPONSE.CREATED',
type: "server",
msg: "RESPONSE.CREATED",
volume_control: device.volume ?? 70,
}));
} else {
ws.send(JSON.stringify({
type: 'server',
msg: 'RESPONSE.CREATED',
type: "server",
msg: "RESPONSE.CREATED",
}));
}
} catch (error) {
console.error('Error fetching device info:', error);
console.error("Error fetching device info:", error);
ws.send(JSON.stringify({
type: 'server',
msg: 'RESPONSE.CREATED',
type: "server",
msg: "RESPONSE.CREATED",
}));
}
createdSent = true;
@ -129,13 +132,13 @@ export const connectToHume = (
try {
// Decode base64 audio data from Hume (this is a WAV file, not raw PCM!)
const wavBuffer = Buffer.from(audioMsg.data, 'base64');
const wavBuffer = Buffer.from(audioMsg.data, "base64");
// Extract PCM data from WAV file
const pcmData = extractPcmFromWav(wavBuffer);
if (!pcmData) {
console.error('Failed to extract PCM data from WAV');
console.error("Failed to extract PCM data from WAV");
return;
}
@ -143,75 +146,56 @@ export const connectToHume = (
const downsampledPcm = downsamplePcm(pcmData, 48000, 24000);
boostLimitPCM16LEInPlace(downsampledPcm, /*gainDb=*/6.0, /*ceiling=*/0.89);
// Process the downsampled PCM data in frames
let audioBuffer = downsampledPcm;
// Process complete frames using the standard FRAME_SIZE
while (audioBuffer.length >= FRAME_SIZE) {
const frame = audioBuffer.subarray(0, FRAME_SIZE);
audioBuffer = audioBuffer.subarray(FRAME_SIZE);
try {
const packet = encoder.encode(frame);
ws.send(packet);
} catch (err) {
console.error('Hume Opus encode failed:', err);
}
}
// Store remaining bytes for next chunk (if any)
if (audioBuffer.length > 0) {
console.log(`Hume audio remainder: ${audioBuffer.length} bytes`);
}
// Use Opus packetizer to encode and send audio
opus.push(downsampledPcm);
} catch (audioError) {
console.error('Error processing Hume audio output:', audioError);
console.error("Error processing Hume audio output:", audioError);
}
break;
case 'chat_metadata':
console.log('Chat metadata received:', message);
case "chat_metadata":
console.log("Chat metadata received:", message);
break;
case 'user_message':
console.log('User message acknowledged:', message);
case "user_message":
console.log("User message acknowledged:", message);
await addConversation(
supabase,
'user',
"user",
message.message.content,
user,
);
break;
case 'user_input':
case "user_input":
// This is an echo of our own input, we can log it but don't need to store it again
console.log('User input acknowledged by Hume');
console.log("User input acknowledged by Hume");
break;
case 'error':
case "error":
const errorMsg = message as HumeError;
console.error(`Hume error: ${errorMsg.code} - ${errorMsg.message}`);
ws.send(JSON.stringify({
type: 'server',
msg: 'RESPONSE.ERROR',
type: "server",
msg: "RESPONSE.ERROR",
error: errorMsg.message,
}));
break;
case 'session_created':
console.log('Hume session created');
case "session_created":
console.log("Hume session created");
ws.send(JSON.stringify({
type: 'server',
msg: 'SESSION.CREATED',
type: "server",
msg: "SESSION.CREATED",
}));
break;
case 'session_ended':
console.log('Hume session ended');
case "session_ended":
console.log("Hume session ended");
ws.send(JSON.stringify({
type: 'server',
msg: 'SESSION.END',
type: "server",
msg: "SESSION.END",
}));
break;
@ -219,31 +203,31 @@ export const connectToHume = (
console.log(`Unhandled Hume message type: ${message.type}`);
}
} catch (error) {
console.error('Error processing Hume message:', error);
console.error("Error processing Hume message:", error);
}
});
humeWs.on('close', (code: number, reason: Buffer) => {
humeWs.on("close", (code: number, reason: Buffer) => {
console.log(`Hume WebSocket closed: ${code} - ${reason.toString()}`);
ws.send(JSON.stringify({
type: 'server',
msg: 'SESSION.END',
type: "server",
msg: "SESSION.END",
}));
isConnected = false;
ws.close();
});
humeWs.on('error', (error: Error) => {
console.error('Hume WebSocket error:', error);
console.error('Error details:', {
humeWs.on("error", (error: Error) => {
console.error("Hume WebSocket error:", error);
console.error("Error details:", {
message: error.message,
stack: error.stack,
name: error.name
});
ws.send(JSON.stringify({
type: 'server',
msg: 'RESPONSE.ERROR',
error: 'Connection to Hume failed',
type: "server",
msg: "RESPONSE.ERROR",
error: "Connection to Hume failed",
}));
});
@ -252,10 +236,10 @@ export const connectToHume = (
try {
if (isBinary) {
// Handle audio data from ESP32
const base64Audio = data.toString('base64');
const base64Audio = data.toString("base64");
const audioMessage: HumeAudioInput = {
type: 'audio_input',
type: "audio_input",
data: base64Audio,
};
@ -269,12 +253,12 @@ export const connectToHume = (
}
}
} catch (error) {
console.error('Error handling message:', error);
console.error("Error handling message:", error);
}
};
// Set up ESP32 WebSocket handlers
ws.on('message', (data: RawData, isBinary: boolean) => {
ws.on("message", (data: RawData, isBinary: boolean) => {
if (!isConnected) {
messageQueue.push(data);
} else {
@ -282,34 +266,35 @@ export const connectToHume = (
}
});
ws.on('error', (error: Error) => {
console.error('ESP32 WebSocket error:', error);
ws.on("error", (error: Error) => {
console.error("ESP32 WebSocket error:", error);
humeWs.close();
});
ws.on('close', async (code: number, reason: string) => {
ws.on("close", async (code: number, reason: string) => {
console.log(`ESP32 WebSocket closed: ${code} - ${reason}`);
humeWs.close();
await closeHandler();
opus.close();
humeWs.close();
if (isDev && connectionPcmFile) {
connectionPcmFile.close();
console.log('Closed debug audio file');
console.log("Closed debug audio file");
}
});
// Wait for Hume connection to be established
return new Promise<void>((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('Hume connection timeout'));
reject(new Error("Hume connection timeout"));
}, 10000);
humeWs.on('open', () => {
humeWs.on("open", () => {
clearTimeout(timeout);
resolve();
});
humeWs.on('error', (error) => {
humeWs.on("error", (error) => {
clearTimeout(timeout);
reject(error);
});

120
server-deno/models/openai.ts
View file

@ -1,14 +1,9 @@
import { Buffer } from "node:buffer";
import type {
RawData,
WebSocket as WSWebSocket,
WebSocketServer as _WebSocketServer,
} from "npm:@types/ws";
import type { RawData } from "npm:@types/ws";
import { RealtimeClient } from "../realtime/client.js";
import { RealtimeUtils } from "../realtime/utils.js";
import { addConversation, getDeviceInfo } from "../supabase.ts";
import { encoder, FRAME_SIZE, isDev, openaiApiKey } from "../utils.ts";
import { createOpusPacketizer, isDev, openaiApiKey, defaultOpenAIVoice } from "../utils.ts";
const sendFirstMessage = (client: RealtimeClient, firstMessage: string) => {
const event = {
@ -32,20 +27,23 @@ const sendFirstMessage = (client: RealtimeClient, firstMessage: string) => {
});
};
export const connectToOpenAI = async (
ws: WebSocket,
payload: IPayload,
connectionPcmFile: Deno.FsFile | null,
firstMessage: string,
systemPrompt: string,
) => {
export const connectToOpenAI = async ({
ws,
payload,
connectionPcmFile,
firstMessage,
systemPrompt,
closeHandler,
}: ProviderArgs) => {
const { user, supabase } = payload;
const opus = createOpusPacketizer((packet) => ws.send(packet));
let currentItemId: string | null = null;
let currentCallId: string | null = null;
// Instantiate new client
console.log(`Connecting with key "${openaiApiKey.slice(0, 3)}..."`);
console.log(`Connecting with key "${openaiApiKey?.slice(0, 3)}..."`);
const client = new RealtimeClient({ apiKey: openaiApiKey });
// ADD TOOL CALLS HERE
@ -73,23 +71,6 @@ export const connectToOpenAI = async (
// Send your custom message to the client
ws.send(JSON.stringify({ type: "server", msg: "SESSION.END" }));
// Send the function result back to OpenAI
const functionResult = {
event_id: RealtimeUtils.generateId("evt_"),
type: "conversation.item.create",
item: {
id: RealtimeUtils.generateId("item_"),
type: "function_call_output",
call_id: currentCallId,
output: JSON.stringify({
success: true,
message: `Session ended: ${args.reason}`,
}),
},
};
client.realtime.send(functionResult.type, functionResult);
// Return the result for the callback
return { success: true, message: `Session ended: ${args.reason}` };
},
@ -97,7 +78,6 @@ export const connectToOpenAI = async (
// Relay: OpenAI Realtime API Event -> Browser Event
client.realtime.on("server.*", async (event: any) => {
// console.log(`Relaying "${event.type}" to Client`);
// Check if the event is session.created
if (event.type === "session.created") {
console.log("session created", event);
@ -107,28 +87,10 @@ export const connectToOpenAI = async (
} else if (event.type === "error") {
console.log("error", event);
} else if (event.type === "response.done") {
// Fetch the latest device info when response is complete
try {
const device = await getDeviceInfo(supabase, user.user_id);
if (device) {
// Send the updated volume data along with the response complete message
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
volume_control: device.volume ?? 100,
}));
} else {
// Fall back to just sending the complete message if there's an error
ws.send(
JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
}),
);
}
} catch (error) {
console.error("Error fetching updated device info:", error);
console.log("response.done", event);
const hasNoAudio = event.response?.usage?.output_token_details?.audio_tokens === 0;
opus.flush(true);
if (!hasNoAudio) {
ws.send(
JSON.stringify({
type: "server",
@ -153,12 +115,35 @@ export const connectToOpenAI = async (
switch (event.type) {
case "response.created":
console.log("response.created", event);
opus.reset();
try {
const device = await getDeviceInfo(supabase, user.user_id);
if (device) {
// Send the updated volume data along with the response complete message
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED",
volume_control: device.volume ?? 100,
}));
} else {
// Fall back to just sending the complete message if there's an error
ws.send(
JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED",
}),
);
}
} catch (error) {
console.error("Error fetching updated device info:", error);
ws.send(
JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED",
}),
);
}
break;
case "response.output_item.added":
console.log("response.output_item.added", event);
@ -178,25 +163,7 @@ export const connectToOpenAI = async (
const pcmBuffer = Buffer.from(
delta.audio.buffer,
);
for (
let offset = 0;
offset < pcmBuffer.length;
offset += FRAME_SIZE
) {
// Get one frame of PCM data.
const frame = pcmBuffer.subarray(
offset,
offset + FRAME_SIZE,
);
try {
const encodedPacket = encoder
.encode(frame);
ws.send(encodedPacket);
} catch (_e) {
// Skip this frame but continue with others
}
}
opus.push(pcmBuffer);
}
} catch (audioError) {
console.error(
@ -326,6 +293,8 @@ export const connectToOpenAI = async (
// Add more detailed close handling
ws.on("close", async (code: number, reason: string) => {
console.log(`WebSocket closed with code ${code}, reason: ${reason}`);
await closeHandler();
opus.close();
client.disconnect();
if (isDev) {
if (connectionPcmFile) {
@ -340,14 +309,13 @@ export const connectToOpenAI = async (
console.log(`Connecting to OpenAI...`);
const sessionOptions = {
model: "gpt-4o-mini-realtime-preview-2024-12-17",
// turn_detection: null,
turn_detection: {
type: "server_vad",
threshold: 0.4,
prefix_padding_ms: 400,
silence_duration_ms: 1000,
},
voice: user.personality?.oai_voice ?? "ash",
voice: user.personality?.oai_voice ?? defaultOpenAIVoice,
instructions: systemPrompt,
input_audio_transcription: { model: "whisper-1" },
};

9
server-deno/types.d.ts vendored
View file

@ -199,4 +199,13 @@ declare global {
code: string;
message: string;
}
interface ProviderArgs {
ws: WebSocket;
payload: IPayload;
connectionPcmFile: Deno.FsFile | null;
firstMessage: string;
systemPrompt: string;
closeHandler: () => Promise<void>;
}
}

79
server-deno/utils.ts
View file

@ -7,6 +7,10 @@ import { Encoder } from "@evan/opus";
export const defaultVolume = 50;
export const defaultGeminiVoice = "Sadachbia";
export const defaultOpenAIVoice = "ash";
export const defaultGrokVoice = "Ara";
// Define your audio parameters
export const SAMPLE_RATE = 24000; // For example, 24000 Hz
const CHANNELS = 1; // Mono (set to 2 if you have stereo)
@ -15,6 +19,79 @@ const BYTES_PER_SAMPLE = 2; // 16-bit PCM: 2 bytes per sample
const FRAME_SIZE = (SAMPLE_RATE * FRAME_DURATION / 1000) * CHANNELS *
BYTES_PER_SAMPLE; // 960 bytes for 24000 Hz mono 16-bit
export function createOpusEncoder() {
const enc = new Encoder({
channels: CHANNELS,
sample_rate: SAMPLE_RATE,
application: "voip",
});
enc.expert_frame_duration = FRAME_DURATION;
enc.bitrate = 24000;
return enc;
}
export function createOpusPacketizer(
sendPacket: (packet: Uint8Array) => void,
) {
const enc = createOpusEncoder();
let pending = Buffer.alloc(0);
let closed = false;
const push = (pcm: Uint8Array) => {
if (closed) return;
if (!pcm || pcm.length === 0) return;
pending = Buffer.concat([pending, Buffer.from(pcm)]);
while (pending.length >= FRAME_SIZE) {
const frame = pending.subarray(0, FRAME_SIZE);
pending = pending.subarray(FRAME_SIZE);
try {
const packet = enc.encode(frame);
sendPacket(packet);
} catch (err) {
console.error("Opus encode failed:", err);
}
}
};
const flush = (padFinalFrame = false) => {
if (closed) return;
if (pending.length === 0) return;
if (!padFinalFrame) {
pending = Buffer.alloc(0);
return;
}
const padded = Buffer.alloc(FRAME_SIZE);
pending.copy(padded, 0, 0, pending.length);
pending = Buffer.alloc(0);
try {
const packet = enc.encode(padded);
sendPacket(packet);
} catch (err) {
console.error("Opus encode failed:", err);
}
};
const reset = () => {
pending = Buffer.alloc(0);
};
const close = () => {
closed = true;
pending = Buffer.alloc(0);
};
const bufferedBytes = () => pending.length;
return { push, flush, reset, close, bufferedBytes };
}
// Legacy encoder for backwards compatibility during migration
const encoder = new Encoder({
channels: CHANNELS,
sample_rate: SAMPLE_RATE,
@ -22,7 +99,7 @@ const encoder = new Encoder({
});
encoder.expert_frame_duration = FRAME_DURATION;
encoder.bitrate = 12000;
encoder.bitrate = 24000;
export const openaiApiKey = Deno.env.get("OPENAI_API_KEY");
export const geminiApiKey = Deno.env.get("GEMINI_API_KEY");