admin-valentin/AIvoices
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

adding eleven labs

Browse source
This commit is contained in:
akdeb 2025-09-19 16:37:11 +07:00
parent eb212672ce
commit 165cd52f1a
9 changed files with 613 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
server-deno/.env.example
View file

@ -8,6 +8,8 @@ ENCRYPTION_KEY=<ENCRYPTION_KEY>
# OpenAI API Key
OPENAI_API_KEY=<OPENAI_API_KEY>
GEMINI_API_KEY=<GEMINI_API_KEY>
ELEVENLABS_API_KEY=<ELEVENLABS_API_KEY>
# local development
HOST=0.0.0.0

83
server-deno/README-elevenlabs.md Normal file
View file

@ -0,0 +1,83 @@
# ElevenLabs WebSocket Relay
This implementation provides a WebSocket relay between your ESP32 device and ElevenLabs Conversational AI API, similar to the existing OpenAI and Gemini implementations.
## Setup
1. **Environment Variables**
Add your ElevenLabs API key to your `.env` file:
```
ELEVENLABS_API_KEY=your_elevenlabs_api_key_here
```
2. **Agent Configuration**
- Create an agent in the ElevenLabs dashboard
- Copy the agent ID
- Store the agent ID in your personality configuration (the `key` field should contain the ElevenLabs agent ID)
3. **Database Configuration**
Update your personality record to use ElevenLabs:
```sql
UPDATE personalities
SET provider = 'elevenlabs', key = 'your_agent_id_here'
WHERE personality_id = 'your_personality_id';
```
## How it Works
1. **Connection Flow**:
- ESP32 connects to your Deno server via WebSocket
- Server authenticates the user and gets their personality configuration
- If provider is "elevenlabs", server requests a signed URL from ElevenLabs API
- Server establishes WebSocket connection to ElevenLabs using the signed URL
- Server acts as a relay between ESP32 and ElevenLabs
2. **Audio Processing**:
- ESP32 sends PCM16 audio data (binary) to server
- Server converts to base64 and forwards to ElevenLabs
- ElevenLabs sends back base64 audio data
- Server converts to PCM16, encodes with Opus, and sends to ESP32
3. **Message Types**:
- **From ElevenLabs**: `audio`, `user_transcript`, `agent_response`, `ping`, `conversation_end`
- **To ElevenLabs**: `audio`, `pong`, `interrupt`
- **To ESP32**: Various server messages like `CONVERSATION.INITIATED`, `RESPONSE.COMPLETE`
## Features
- ✅ Real-time audio streaming
- ✅ Automatic transcription (user and agent)
- ✅ Conversation history storage
- ✅ Error handling and reconnection
- ✅ Debug audio file logging (in dev mode)
- ✅ Volume control integration
- ✅ Ping/pong keepalive
## Usage
Once configured, the ElevenLabs provider works exactly like OpenAI and Gemini:
```typescript
// In your personality configuration
{
provider: "elevenlabs",
key: "your_elevenlabs_agent_id",
// other personality fields...
}
```
The server will automatically route to the ElevenLabs implementation when the provider is set to "elevenlabs".
## Differences from OpenAI/Gemini
- **No system prompts**: ElevenLabs agents are configured in their dashboard
- **No first message**: Agent behavior is defined in ElevenLabs configuration
- **Automatic VAD**: ElevenLabs handles voice activity detection
- **Agent-based**: Uses pre-configured agents rather than model + prompt
## Troubleshooting
1. **Connection Issues**: Ensure your ElevenLabs API key is valid and has access to Conversational AI
2. **Agent Not Found**: Verify the agent ID is correct and the agent exists in your ElevenLabs account
3. **Audio Issues**: Check that your ESP32 is sending PCM16 audio at 24kHz sample rate
4. **Transcription Missing**: Ensure your ElevenLabs agent has transcription enabled in the dashboard

1
server-deno/deno.json
View file

@ -33,6 +33,7 @@
}
},
"imports": {
"@elevenlabs/client": "npm:@elevenlabs/client@^0.6.2",
"@evan/opus": "npm:@evan/opus@^1.0.3",
"@std/assert": "jsr:@std/assert@1",
"@supabase/supabase-js": "jsr:@supabase/supabase-js@^2.48.1"

163
server-deno/deno.lock
View file

@ -1,15 +1,24 @@
{
"version": "5",
"specifiers": {
"jsr:@std/assert@1": "1.0.14",
"jsr:@std/internal@^1.0.10": "1.0.10",
"jsr:@supabase/supabase-js@2": "2.48.1",
"jsr:@supabase/supabase-js@^2.48.1": "2.48.1",
"jsr:@supabase/supabase-js@^2.48.1": "2.57.4",
"npm:@elevenlabs/client@*": "0.6.2",
"npm:@elevenlabs/client@~0.6.2": "0.6.2",
"npm:@evan/opus@^1.0.3": "1.0.3",
"npm:@google/genai@*": "1.2.0_@modelcontextprotocol+sdk@1.12.1__express@5.1.0__zod@3.25.36_zod@3.25.36",
"npm:@supabase/auth-js@2.67.3": "2.67.3",
"npm:@supabase/auth-js@2.71.1": "2.71.1",
"npm:@supabase/functions-js@2.4.4": "2.4.4",
"npm:@supabase/functions-js@2.4.6": "2.4.6",
"npm:@supabase/node-fetch@2.6.15": "2.6.15",
"npm:@supabase/postgrest-js@1.18.1": "1.18.1",
"npm:@supabase/postgrest-js@1.21.4": "1.21.4",
"npm:@supabase/realtime-js@2.11.2": "2.11.2",
"npm:@supabase/realtime-js@2.15.5": "2.15.5",
"npm:@supabase/storage-js@2.12.1": "2.12.1",
"npm:@supabase/storage-js@2.7.1": "2.7.1",
"npm:@types/node@*": "22.13.0",
"npm:@types/ws@*": "8.5.14",
@ -17,19 +26,48 @@
"npm:ws@*": "8.18.0"
},
"jsr": {
"@std/assert@1.0.14": {
"integrity": "68d0d4a43b365abc927f45a9b85c639ea18a9fab96ad92281e493e4ed84abaa4",
"dependencies": [
"jsr:@std/internal"
]
},
"@std/internal@1.0.10": {
"integrity": "e3be62ce42cab0e177c27698e5d9800122f67b766a0bea6ca4867886cbde8cf7"
},
"@supabase/supabase-js@2.48.1": {
"integrity": "747e85c2a546efcc9f84b343b08e10e3818d468ae214506b59dc65a36fd2d702",
"dependencies": [
"npm:@supabase/auth-js",
"npm:@supabase/functions-js",
"npm:@supabase/auth-js@2.67.3",
"npm:@supabase/functions-js@2.4.4",
"npm:@supabase/node-fetch",
"npm:@supabase/postgrest-js",
"npm:@supabase/realtime-js",
"npm:@supabase/storage-js"
"npm:@supabase/postgrest-js@1.18.1",
"npm:@supabase/realtime-js@2.11.2",
"npm:@supabase/storage-js@2.7.1"
]
},
"@supabase/supabase-js@2.57.4": {
"integrity": "fc848eabd844e5ffe8f61468354ef5c305eb3cddf60ddd6cef2e20bb54352471",
"dependencies": [
"npm:@supabase/auth-js@2.71.1",
"npm:@supabase/functions-js@2.4.6",
"npm:@supabase/node-fetch",
"npm:@supabase/postgrest-js@1.21.4",
"npm:@supabase/realtime-js@2.15.5",
"npm:@supabase/storage-js@2.12.1"
]
}
},
"npm": {
"@bufbuild/protobuf@1.10.1": {
"integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ=="
},
"@elevenlabs/client@0.6.2": {
"integrity": "sha512-4b3GCZLBINujAQ7PXMfLR4JrffiUc17plak4sXM0KMxxsDtFSrVAUCQHxFiEzDkSpTrMN+JtJbx7zhjIfkpWjA==",
"dependencies": [
"livekit-client"
]
},
"@evan/opus@1.0.3": {
"integrity": "sha512-ADfwIad83W1LuiZDNMjDMDNQRsPz8rj5xnDLExhVWTnA5wGJCLntOn12Ir5rxGBqdfo10QhnNVdd2+gXiZ6xCg=="
},
@ -38,11 +76,20 @@
"dependencies": [
"@modelcontextprotocol/sdk",
"google-auth-library",
"ws",
"ws@8.18.0",
"zod",
"zod-to-json-schema"
]
},
"@livekit/mutex@1.1.1": {
"integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw=="
},
"@livekit/protocol@1.39.3": {
"integrity": "sha512-hfOnbwPCeZBEvMRdRhU2sr46mjGXavQcrb3BFRfG+Gm0Z7WUSeFdy5WLstXJzEepz17Iwp/lkGwJ4ZgOOYfPuA==",
"dependencies": [
"@bufbuild/protobuf"
]
},
"@modelcontextprotocol/sdk@1.12.1_express@5.1.0_zod@3.25.36": {
"integrity": "sha512-KG1CZhZfWg+u8pxeM/mByJDScJSrjjxLc8fwQqbsS8xCjBmQfMNEBTotYdNanKekepnfRI85GtgQlctLFpcYPw==",
"dependencies": [
@ -65,12 +112,24 @@
"@supabase/node-fetch"
]
},
"@supabase/auth-js@2.71.1": {
"integrity": "sha512-mMIQHBRc+SKpZFRB2qtupuzulaUhFYupNyxqDj5Jp/LyPvcWvjaJzZzObv6URtL/O6lPxkanASnotGtNpS3H2Q==",
"dependencies": [
"@supabase/node-fetch"
]
},
"@supabase/functions-js@2.4.4": {
"integrity": "sha512-WL2p6r4AXNGwop7iwvul2BvOtuJ1YQy8EbOd0dhG1oN1q8el/BIRSFCFnWAMM/vJJlHWLi4ad22sKbKr9mvjoA==",
"dependencies": [
"@supabase/node-fetch"
]
},
"@supabase/functions-js@2.4.6": {
"integrity": "sha512-bhjZ7rmxAibjgmzTmQBxJU6ZIBCCJTc3Uwgvdi4FewueUTAGO5hxZT1Sj6tiD+0dSXf9XI87BDdJrg12z8Uaew==",
"dependencies": [
"@supabase/node-fetch"
]
},
"@supabase/node-fetch@2.6.15": {
"integrity": "sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==",
"dependencies": [
@ -83,13 +142,34 @@
"@supabase/node-fetch"
]
},
"@supabase/postgrest-js@1.21.4": {
"integrity": "sha512-TxZCIjxk6/dP9abAi89VQbWWMBbybpGWyvmIzTd79OeravM13OjR/YEYeyUOPcM1C3QyvXkvPZhUfItvmhY1IQ==",
"dependencies": [
"@supabase/node-fetch"
]
},
"@supabase/realtime-js@2.11.2": {
"integrity": "sha512-u/XeuL2Y0QEhXSoIPZZwR6wMXgB+RQbJzG9VErA3VghVt7uRfSVsjeqd7m5GhX3JR6dM/WRmLbVR8URpDWG4+w==",
"dependencies": [
"@supabase/node-fetch",
"@types/phoenix",
"@types/ws",
"ws"
"@types/ws@8.5.14",
"ws@8.18.0"
]
},
"@supabase/realtime-js@2.15.5": {
"integrity": "sha512-/Rs5Vqu9jejRD8ZeuaWXebdkH+J7V6VySbCZ/zQM93Ta5y3mAmocjioa/nzlB6qvFmyylUgKVS1KpE212t30OA==",
"dependencies": [
"@supabase/node-fetch",
"@types/phoenix",
"@types/ws@8.18.1",
"ws@8.18.3"
]
},
"@supabase/storage-js@2.12.1": {
"integrity": "sha512-QWg3HV6Db2J81VQx0PqLq0JDBn4Q8B1FYn1kYcbla8+d5WDmTdwwMr+EJAxNOSs9W4mhKMv+EYCpCrTFlTj4VQ==",
"dependencies": [
"@supabase/node-fetch"
]
},
"@supabase/storage-js@2.7.1": {
@ -98,6 +178,9 @@
"@supabase/node-fetch"
]
},
"@types/dom-mediacapture-record@1.0.22": {
"integrity": "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw=="
},
"@types/node@22.13.0": {
"integrity": "sha512-ClIbNe36lawluuvq3+YYhnIN2CELi+6q8NpnM7PYp4hBn/TatfboPgVSm2rwKRfnV2M+Ty9GWDFI64KEe+kysA==",
"dependencies": [
@ -107,6 +190,12 @@
"@types/phoenix@1.6.6": {
"integrity": "sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A=="
},
"@types/ws@8.18.1": {
"integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==",
"dependencies": [
"@types/node"
]
},
"@types/ws@8.5.14": {
"integrity": "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==",
"dependencies": [
@ -249,6 +338,9 @@
"etag@1.8.1": {
"integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg=="
},
"events@3.3.0": {
"integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q=="
},
"eventsource-parser@3.0.2": {
"integrity": "sha512-6RxOBZ/cYgd8usLwsEl+EC09Au/9BcmCKYF2/xbml6DNczf7nv0MQb+7BA2F+li6//I+28VNlQR37XfQtcAJuA=="
},
@ -460,6 +552,24 @@
"safe-buffer"
]
},
"livekit-client@2.15.7_@types+dom-mediacapture-record@1.0.22": {
"integrity": "sha512-19m8Q1cvRl5PslRawDUgWXeP8vL8584tX8kiZEJaPZo83U/L6VPS/O7pP06phfJaBWeeV8sAOVtEPlQiZEHtpg==",
"dependencies": [
"@livekit/mutex",
"@livekit/protocol",
"@types/dom-mediacapture-record",
"events",
"loglevel",
"sdp-transform",
"ts-debounce",
"tslib",
"typed-emitter",
"webrtc-adapter"
]
},
"loglevel@1.9.2": {
"integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg=="
},
"math-intrinsics@1.1.0": {
"integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g=="
},
@ -558,12 +668,25 @@
"path-to-regexp"
]
},
"rxjs@7.8.2": {
"integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==",
"dependencies": [
"tslib"
]
},
"safe-buffer@5.2.1": {
"integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ=="
},
"safer-buffer@2.1.2": {
"integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
},
"sdp-transform@2.15.0": {
"integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==",
"bin": true
},
"sdp@3.2.1": {
"integrity": "sha512-lwsAIzOPlH8/7IIjjz3K0zYBk7aBVVcvjMwt3M4fLxpjMYyy7i3I97SLHebgn4YBjirkzfp3RvRDWSKsh/+WFw=="
},
"send@1.2.0": {
"integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==",
"dependencies": [
@ -646,6 +769,12 @@
"tr46@0.0.3": {
"integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw=="
},
"ts-debounce@4.0.0": {
"integrity": "sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg=="
},
"tslib@2.8.1": {
"integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="
},
"type-is@2.0.1": {
"integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==",
"dependencies": [
@ -654,6 +783,12 @@
"mime-types"
]
},
"typed-emitter@2.1.0": {
"integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==",
"optionalDependencies": [
"rxjs"
]
},
"undici-types@6.20.0": {
"integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg=="
},
@ -680,6 +815,12 @@
"webidl-conversions@3.0.1": {
"integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ=="
},
"webrtc-adapter@9.0.3": {
"integrity": "sha512-5fALBcroIl31OeXAdd1YUntxiZl1eHlZZWzNg3U4Fn+J9/cGL3eT80YlrsWGvj2ojuz1rZr2OXkgCzIxAZ7vRQ==",
"dependencies": [
"sdp"
]
},
"whatwg-url@5.0.0": {
"integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
"dependencies": [
@ -700,6 +841,9 @@
"ws@8.18.0": {
"integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw=="
},
"ws@8.18.3": {
"integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg=="
},
"zod-to-json-schema@3.24.5_zod@3.25.36": {
"integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==",
"dependencies": [
@ -796,6 +940,7 @@
"dependencies": [
"jsr:@std/assert@1",
"jsr:@supabase/supabase-js@^2.48.1",
"npm:@elevenlabs/client@~0.6.2",
"npm:@evan/opus@^1.0.3"
]
}

1
server-deno/logs.log Normal file
View file

@ -0,0 +1 @@
Audio capture server running on ws://0.0.0.0:8000

18
server-deno/main.ts
View file

@ -4,7 +4,7 @@ import type {
WebSocket as WSWebSocket,
WebSocketServer as _WebSocketServer,
} from "npm:@types/ws";
import { authenticateUser } from "./utils.ts";
import { authenticateUser, elevenLabsApiKey } from "./utils.ts";
import {
createFirstMessage,
createSystemPrompt,
@ -15,6 +15,7 @@ import { SupabaseClient } from "@supabase/supabase-js";
import { isDev } from "./utils.ts";
import { connectToOpenAI } from "./models/openai.ts";
import { connectToGemini } from "./models/gemini.ts";
import { connectToElevenLabs } from "./models/elevenlabs.ts";
const server = createServer();
@ -75,6 +76,21 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
systemPrompt,
);
break;
case "elevenlabs":
const agentId = user.personality?.oai_voice ?? "";
if (!elevenLabsApiKey) {
throw new Error("ELEVENLABS_API_KEY environment variable is required");
}
await connectToElevenLabs(
ws,
payload,
connectionPcmFile,
agentId,
elevenLabsApiKey,
);
break;
default:
throw new Error(`Unknown provider: ${provider}`);
}

348
server-deno/models/elevenlabs.ts Normal file
View file

@ -0,0 +1,348 @@
import { Buffer } from "node:buffer";
import type { RawData } from "npm:@types/ws";
// @ts-ignore
import {
WebSocketConnection,
type SessionConfig,
type IncomingSocketEvent,
type DisconnectionDetails
} from "npm:@elevenlabs/client";
import { addConversation, getDeviceInfo } from "../supabase.ts";
import { encoder, FRAME_SIZE, isDev } from "../utils.ts";
import type { IPayload } from "../types.d.ts";
// Calculate audio level for debugging
function calculateAudioLevel(audioData: any): number {
if (!audioData || audioData.length === 0) return 0;
// Convert to 16-bit samples
const samples = new Int16Array(audioData.buffer || audioData);
let sum = 0;
for (let i = 0; i < samples.length; i++) {
sum += Math.abs(samples[i]);
}
return Math.round(sum / samples.length);
}
export const connectToElevenLabs = async (
ws: WebSocket,
payload: IPayload,
connectionPcmFile: Deno.FsFile | null,
agentId: string,
apiKey: string,
) => {
console.log(apiKey, agentId);
const { user, supabase } = payload;
// Queue messages until ElevenLabs connection is ready
const messageQueue: RawData[] = [];
let isElevenLabsConnected = false;
let elevenLabsConnection: WebSocketConnection | null = null;
let hasResponseStarted = false;
// Handle messages from ESP32 client
const handleClientMessage = async (data: any, isBinary: boolean) => {
try {
if (isBinary) {
const base64Data = data.toString("base64");
if (isDev && connectionPcmFile) {
await connectionPcmFile.write(data);
}
// Send audio to ElevenLabs using their client
if (isElevenLabsConnected && elevenLabsConnection) {
// Check if audio contains actual speech (simple volume check)
const audioLevel = calculateAudioLevel(data);
console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`);
try {
elevenLabsConnection.sendMessage({
user_audio_chunk: base64Data,
});
} catch (error) {
console.error("Error sending audio to ElevenLabs:", error);
}
} else {
console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`);
}
} else {
const message = JSON.parse(data.toString("utf-8"));
if (message.type === "instruction") {
switch (message.msg) {
case "INTERRUPT":
console.log("Interrupt detected");
if (elevenLabsConnection) {
elevenLabsConnection.sendMessage({
type: "user_activity"
});
}
break;
case "END_SESSION":
console.log("End session requested");
if (elevenLabsConnection) {
elevenLabsConnection.close();
}
break;
}
}
}
} catch (error) {
console.error("Error handling client message:", error);
}
};
try {
// For server-side usage, we need to get a signed URL first
const signedUrlResponse = await fetch(
`https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`,
{
headers: {
'xi-api-key': apiKey,
},
}
);
if (!signedUrlResponse.ok) {
throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`);
}
const { signed_url } = await signedUrlResponse.json();
// Use default audio formats (let ElevenLabs auto-detect)
const modifiedSignedUrl = signed_url;
// Create ElevenLabs connection using signed URL for server-side usage
const sessionConfig: SessionConfig = {
signedUrl: modifiedSignedUrl,
connectionType: "websocket",
};
elevenLabsConnection = await WebSocketConnection.create(sessionConfig);
console.log("Connected to ElevenLabs successfully!");
isElevenLabsConnected = true;
console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`);
// Send initial RESPONSE.CREATED for the first message
console.log("Sending initial RESPONSE.CREATED to ESP32");
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED"
}));
// Set up ElevenLabs event handlers
elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => {
console.log("ElevenLabs message type:", event.type);
switch (event.type) {
case "conversation_initiation_metadata":
console.log("ElevenLabs conversation initiated (metadata received)");
// RESPONSE.CREATED already sent when connection was established
break;
case "ping":
// Handle ping messages - send pong response
console.log("Received ping from ElevenLabs, sending pong");
if (event.ping_event?.event_id) {
elevenLabsConnection.sendMessage({
type: "pong",
event_id: event.ping_event.event_id
});
}
break;
case "audio":
if (event.audio_event?.audio_base_64) {
// Send RESPONSE.CREATED only for the first audio chunk of each response
if (!hasResponseStarted) {
console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)");
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.CREATED"
}));
hasResponseStarted = true;
}
const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64");
console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes, processing into ${Math.ceil(audioBuffer.length / FRAME_SIZE)} frames`);
let framesSent = 0;
// Process audio in frames for Opus encoding
for (let offset = 0; offset < audioBuffer.length; offset += FRAME_SIZE) {
const frame = audioBuffer.subarray(offset, offset + FRAME_SIZE);
try {
const encodedPacket = encoder.encode(frame);
ws.send(encodedPacket);
framesSent++;
} catch (_e) {
// Skip this frame but continue with others
console.log(`Failed to encode frame at offset ${offset}`);
}
}
console.log(`Sent ${framesSent} audio frames to ESP32`);
}
break;
case "user_transcript":
if (event.user_transcription_event?.user_transcript) {
console.log("User transcript:", event.user_transcription_event.user_transcript);
addConversation(
supabase,
"user",
event.user_transcription_event.user_transcript,
user,
);
// Send audio committed message like OpenAI does
console.log("Sending AUDIO.COMMITTED to ESP32");
ws.send(JSON.stringify({
type: "server",
msg: "AUDIO.COMMITTED"
}));
}
break;
case "agent_response":
if (event.agent_response_event?.agent_response) {
console.log("Agent response:", event.agent_response_event.agent_response);
addConversation(
supabase,
"assistant",
event.agent_response_event.agent_response,
user,
);
// Send response complete with device info like OpenAI does
console.log("Sending RESPONSE.COMPLETE to ESP32");
hasResponseStarted = false; // Reset for next response
try {
const device = await getDeviceInfo(supabase, user.user_id);
if (device) {
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
volume_control: device.volume ?? 100,
}));
} else {
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
}));
}
} catch (error) {
console.error("Error fetching updated device info:", error);
ws.send(JSON.stringify({
type: "server",
msg: "RESPONSE.COMPLETE",
}));
}
}
break;
case "vad_score":
// Voice Activity Detection score - can be used for debugging
if (event.vad_score_event?.vad_score) {
console.log("VAD score:", event.vad_score_event.vad_score);
}
break;
case "internal_tentative_agent_response":
// Tentative response while agent is thinking
if (event.tentative_agent_response_internal_event?.tentative_agent_response) {
console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response);
}
break;
case "conversation_end":
console.log("ElevenLabs conversation ended");
ws.send(JSON.stringify({
type: "server",
msg: "SESSION.END"
}));
break;
default:
console.log("Unknown ElevenLabs message:", event.type, event);
}
});
elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => {
console.log("ElevenLabs connection closed:", details.reason);
ws.close();
});
// Process queued messages
while (messageQueue.length > 0) {
const queuedMessage = messageQueue.shift();
if (queuedMessage) {
handleClientMessage(queuedMessage, false);
}
}
// Set up ESP32 WebSocket handlers
ws.on("message", (data: any, isBinary: boolean) => {
if (!isElevenLabsConnected) {
messageQueue.push(data);
} else {
handleClientMessage(data, isBinary);
}
});
ws.on("error", (error: any) => {
console.error("ESP32 WebSocket error:", error);
elevenLabsConnection?.close();
});
ws.on("close", async (code: number, reason: string) => {
console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`);
elevenLabsConnection?.close();
if (isDev && connectionPcmFile) {
connectionPcmFile.close();
console.log("Closed debug audio file.");
}
});
// Send initial device info to ESP32
try {
const device = await getDeviceInfo(supabase, user.user_id);
if (device) {
ws.send(JSON.stringify({
type: "server",
msg: "DEVICE.INFO",
volume_control: device.volume ?? 100,
}));
}
} catch (error) {
console.error("Error fetching device info:", error);
}
} catch (error) {
console.error("Failed to connect to ElevenLabs:", error);
// Send more specific error information
let errorMessage = "RESPONSE.ERROR";
if (error instanceof Error) {
console.error("Error details:", error.message);
if (error.message.includes("signed URL")) {
errorMessage = "AUTH.ERROR";
}
}
ws.send(JSON.stringify({
type: "server",
msg: errorMessage
}));
}
};

8
server-deno/types.d.ts vendored
View file

@ -26,7 +26,7 @@ declare global {
user_code: string;
}
type ModelProvider = "openai" | "gemini";
type ModelProvider = "openai" | "gemini" | "elevenlabs";
type GeminiVoice =
| "Zephyr"
@ -70,13 +70,17 @@ declare global {
| "sage"
| "verse";
/**
* Note: oai_voice is essentially the name of the voice.
* the naming here sucks, please change it
*/
interface IPersonality {
personality_id: string;
is_doctor: boolean;
is_child_voice: boolean;
is_story: boolean;
key: string;
oai_voice: OaiVoice | GeminiVoice;
oai_voice: string;
provider: ModelProvider;
voice_description: string;
title: string;

1
server-deno/utils.ts
View file

@ -26,6 +26,7 @@ encoder.bitrate = 12000;
export const openaiApiKey = Deno.env.get("OPENAI_API_KEY");
export const geminiApiKey = Deno.env.get("GEMINI_API_KEY");
export const elevenLabsApiKey = Deno.env.get("ELEVENLABS_API_KEY");
export { encoder, FRAME_SIZE };