From 165cd52f1a4e166774ac0bed468134adf86894b3 Mon Sep 17 00:00:00 2001 From: akdeb Date: Fri, 19 Sep 2025 16:37:11 +0700 Subject: [PATCH] adding eleven labs --- server-deno/.env.example | 2 + server-deno/README-elevenlabs.md | 83 ++++++++ server-deno/deno.json | 1 + server-deno/deno.lock | 163 ++++++++++++++- server-deno/logs.log | 1 + server-deno/main.ts | 18 +- server-deno/models/elevenlabs.ts | 348 +++++++++++++++++++++++++++++++ server-deno/types.d.ts | 8 +- server-deno/utils.ts | 1 + 9 files changed, 613 insertions(+), 12 deletions(-) create mode 100644 server-deno/README-elevenlabs.md create mode 100644 server-deno/logs.log create mode 100644 server-deno/models/elevenlabs.ts diff --git a/server-deno/.env.example b/server-deno/.env.example index eae4406..2bf18b5 100644 --- a/server-deno/.env.example +++ b/server-deno/.env.example @@ -8,6 +8,8 @@ ENCRYPTION_KEY= # OpenAI API Key OPENAI_API_KEY= +GEMINI_API_KEY= +ELEVENLABS_API_KEY= # local development HOST=0.0.0.0 diff --git a/server-deno/README-elevenlabs.md b/server-deno/README-elevenlabs.md new file mode 100644 index 0000000..d5cd2d5 --- /dev/null +++ b/server-deno/README-elevenlabs.md @@ -0,0 +1,83 @@ +# ElevenLabs WebSocket Relay + +This implementation provides a WebSocket relay between your ESP32 device and ElevenLabs Conversational AI API, similar to the existing OpenAI and Gemini implementations. + +## Setup + +1. **Environment Variables** + Add your ElevenLabs API key to your `.env` file: + ``` + ELEVENLABS_API_KEY=your_elevenlabs_api_key_here + ``` + +2. **Agent Configuration** + - Create an agent in the ElevenLabs dashboard + - Copy the agent ID + - Store the agent ID in your personality configuration (the `key` field should contain the ElevenLabs agent ID) + +3. **Database Configuration** + Update your personality record to use ElevenLabs: + ```sql + UPDATE personalities + SET provider = 'elevenlabs', key = 'your_agent_id_here' + WHERE personality_id = 'your_personality_id'; + ``` + +## How it Works + +1. **Connection Flow**: + - ESP32 connects to your Deno server via WebSocket + - Server authenticates the user and gets their personality configuration + - If provider is "elevenlabs", server requests a signed URL from ElevenLabs API + - Server establishes WebSocket connection to ElevenLabs using the signed URL + - Server acts as a relay between ESP32 and ElevenLabs + +2. **Audio Processing**: + - ESP32 sends PCM16 audio data (binary) to server + - Server converts to base64 and forwards to ElevenLabs + - ElevenLabs sends back base64 audio data + - Server converts to PCM16, encodes with Opus, and sends to ESP32 + +3. **Message Types**: + - **From ElevenLabs**: `audio`, `user_transcript`, `agent_response`, `ping`, `conversation_end` + - **To ElevenLabs**: `audio`, `pong`, `interrupt` + - **To ESP32**: Various server messages like `CONVERSATION.INITIATED`, `RESPONSE.COMPLETE` + +## Features + +- ✅ Real-time audio streaming +- ✅ Automatic transcription (user and agent) +- ✅ Conversation history storage +- ✅ Error handling and reconnection +- ✅ Debug audio file logging (in dev mode) +- ✅ Volume control integration +- ✅ Ping/pong keepalive + +## Usage + +Once configured, the ElevenLabs provider works exactly like OpenAI and Gemini: + +```typescript +// In your personality configuration +{ + provider: "elevenlabs", + key: "your_elevenlabs_agent_id", + // other personality fields... +} +``` + +The server will automatically route to the ElevenLabs implementation when the provider is set to "elevenlabs". + +## Differences from OpenAI/Gemini + +- **No system prompts**: ElevenLabs agents are configured in their dashboard +- **No first message**: Agent behavior is defined in ElevenLabs configuration +- **Automatic VAD**: ElevenLabs handles voice activity detection +- **Agent-based**: Uses pre-configured agents rather than model + prompt + +## Troubleshooting + +1. **Connection Issues**: Ensure your ElevenLabs API key is valid and has access to Conversational AI +2. **Agent Not Found**: Verify the agent ID is correct and the agent exists in your ElevenLabs account +3. **Audio Issues**: Check that your ESP32 is sending PCM16 audio at 24kHz sample rate +4. **Transcription Missing**: Ensure your ElevenLabs agent has transcription enabled in the dashboard \ No newline at end of file diff --git a/server-deno/deno.json b/server-deno/deno.json index 13b4c3a..6301142 100644 --- a/server-deno/deno.json +++ b/server-deno/deno.json @@ -33,6 +33,7 @@ } }, "imports": { + "@elevenlabs/client": "npm:@elevenlabs/client@^0.6.2", "@evan/opus": "npm:@evan/opus@^1.0.3", "@std/assert": "jsr:@std/assert@1", "@supabase/supabase-js": "jsr:@supabase/supabase-js@^2.48.1" diff --git a/server-deno/deno.lock b/server-deno/deno.lock index 3a6b796..1f2f25b 100644 --- a/server-deno/deno.lock +++ b/server-deno/deno.lock @@ -1,15 +1,24 @@ { "version": "5", "specifiers": { + "jsr:@std/assert@1": "1.0.14", + "jsr:@std/internal@^1.0.10": "1.0.10", "jsr:@supabase/supabase-js@2": "2.48.1", - "jsr:@supabase/supabase-js@^2.48.1": "2.48.1", + "jsr:@supabase/supabase-js@^2.48.1": "2.57.4", + "npm:@elevenlabs/client@*": "0.6.2", + "npm:@elevenlabs/client@~0.6.2": "0.6.2", "npm:@evan/opus@^1.0.3": "1.0.3", "npm:@google/genai@*": "1.2.0_@modelcontextprotocol+sdk@1.12.1__express@5.1.0__zod@3.25.36_zod@3.25.36", "npm:@supabase/auth-js@2.67.3": "2.67.3", + "npm:@supabase/auth-js@2.71.1": "2.71.1", "npm:@supabase/functions-js@2.4.4": "2.4.4", + "npm:@supabase/functions-js@2.4.6": "2.4.6", "npm:@supabase/node-fetch@2.6.15": "2.6.15", "npm:@supabase/postgrest-js@1.18.1": "1.18.1", + "npm:@supabase/postgrest-js@1.21.4": "1.21.4", "npm:@supabase/realtime-js@2.11.2": "2.11.2", + "npm:@supabase/realtime-js@2.15.5": "2.15.5", + "npm:@supabase/storage-js@2.12.1": "2.12.1", "npm:@supabase/storage-js@2.7.1": "2.7.1", "npm:@types/node@*": "22.13.0", "npm:@types/ws@*": "8.5.14", @@ -17,19 +26,48 @@ "npm:ws@*": "8.18.0" }, "jsr": { + "@std/assert@1.0.14": { + "integrity": "68d0d4a43b365abc927f45a9b85c639ea18a9fab96ad92281e493e4ed84abaa4", + "dependencies": [ + "jsr:@std/internal" + ] + }, + "@std/internal@1.0.10": { + "integrity": "e3be62ce42cab0e177c27698e5d9800122f67b766a0bea6ca4867886cbde8cf7" + }, "@supabase/supabase-js@2.48.1": { "integrity": "747e85c2a546efcc9f84b343b08e10e3818d468ae214506b59dc65a36fd2d702", "dependencies": [ - "npm:@supabase/auth-js", - "npm:@supabase/functions-js", + "npm:@supabase/auth-js@2.67.3", + "npm:@supabase/functions-js@2.4.4", "npm:@supabase/node-fetch", - "npm:@supabase/postgrest-js", - "npm:@supabase/realtime-js", - "npm:@supabase/storage-js" + "npm:@supabase/postgrest-js@1.18.1", + "npm:@supabase/realtime-js@2.11.2", + "npm:@supabase/storage-js@2.7.1" + ] + }, + "@supabase/supabase-js@2.57.4": { + "integrity": "fc848eabd844e5ffe8f61468354ef5c305eb3cddf60ddd6cef2e20bb54352471", + "dependencies": [ + "npm:@supabase/auth-js@2.71.1", + "npm:@supabase/functions-js@2.4.6", + "npm:@supabase/node-fetch", + "npm:@supabase/postgrest-js@1.21.4", + "npm:@supabase/realtime-js@2.15.5", + "npm:@supabase/storage-js@2.12.1" ] } }, "npm": { + "@bufbuild/protobuf@1.10.1": { + "integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==" + }, + "@elevenlabs/client@0.6.2": { + "integrity": "sha512-4b3GCZLBINujAQ7PXMfLR4JrffiUc17plak4sXM0KMxxsDtFSrVAUCQHxFiEzDkSpTrMN+JtJbx7zhjIfkpWjA==", + "dependencies": [ + "livekit-client" + ] + }, "@evan/opus@1.0.3": { "integrity": "sha512-ADfwIad83W1LuiZDNMjDMDNQRsPz8rj5xnDLExhVWTnA5wGJCLntOn12Ir5rxGBqdfo10QhnNVdd2+gXiZ6xCg==" }, @@ -38,11 +76,20 @@ "dependencies": [ "@modelcontextprotocol/sdk", "google-auth-library", - "ws", + "ws@8.18.0", "zod", "zod-to-json-schema" ] }, + "@livekit/mutex@1.1.1": { + "integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==" + }, + "@livekit/protocol@1.39.3": { + "integrity": "sha512-hfOnbwPCeZBEvMRdRhU2sr46mjGXavQcrb3BFRfG+Gm0Z7WUSeFdy5WLstXJzEepz17Iwp/lkGwJ4ZgOOYfPuA==", + "dependencies": [ + "@bufbuild/protobuf" + ] + }, "@modelcontextprotocol/sdk@1.12.1_express@5.1.0_zod@3.25.36": { "integrity": "sha512-KG1CZhZfWg+u8pxeM/mByJDScJSrjjxLc8fwQqbsS8xCjBmQfMNEBTotYdNanKekepnfRI85GtgQlctLFpcYPw==", "dependencies": [ @@ -65,12 +112,24 @@ "@supabase/node-fetch" ] }, + "@supabase/auth-js@2.71.1": { + "integrity": "sha512-mMIQHBRc+SKpZFRB2qtupuzulaUhFYupNyxqDj5Jp/LyPvcWvjaJzZzObv6URtL/O6lPxkanASnotGtNpS3H2Q==", + "dependencies": [ + "@supabase/node-fetch" + ] + }, "@supabase/functions-js@2.4.4": { "integrity": "sha512-WL2p6r4AXNGwop7iwvul2BvOtuJ1YQy8EbOd0dhG1oN1q8el/BIRSFCFnWAMM/vJJlHWLi4ad22sKbKr9mvjoA==", "dependencies": [ "@supabase/node-fetch" ] }, + "@supabase/functions-js@2.4.6": { + "integrity": "sha512-bhjZ7rmxAibjgmzTmQBxJU6ZIBCCJTc3Uwgvdi4FewueUTAGO5hxZT1Sj6tiD+0dSXf9XI87BDdJrg12z8Uaew==", + "dependencies": [ + "@supabase/node-fetch" + ] + }, "@supabase/node-fetch@2.6.15": { "integrity": "sha512-1ibVeYUacxWYi9i0cf5efil6adJ9WRyZBLivgjs+AUpewx1F3xPi7gLgaASI2SmIQxPoCEjAsLAzKPgMJVgOUQ==", "dependencies": [ @@ -83,13 +142,34 @@ "@supabase/node-fetch" ] }, + "@supabase/postgrest-js@1.21.4": { + "integrity": "sha512-TxZCIjxk6/dP9abAi89VQbWWMBbybpGWyvmIzTd79OeravM13OjR/YEYeyUOPcM1C3QyvXkvPZhUfItvmhY1IQ==", + "dependencies": [ + "@supabase/node-fetch" + ] + }, "@supabase/realtime-js@2.11.2": { "integrity": "sha512-u/XeuL2Y0QEhXSoIPZZwR6wMXgB+RQbJzG9VErA3VghVt7uRfSVsjeqd7m5GhX3JR6dM/WRmLbVR8URpDWG4+w==", "dependencies": [ "@supabase/node-fetch", "@types/phoenix", - "@types/ws", - "ws" + "@types/ws@8.5.14", + "ws@8.18.0" + ] + }, + "@supabase/realtime-js@2.15.5": { + "integrity": "sha512-/Rs5Vqu9jejRD8ZeuaWXebdkH+J7V6VySbCZ/zQM93Ta5y3mAmocjioa/nzlB6qvFmyylUgKVS1KpE212t30OA==", + "dependencies": [ + "@supabase/node-fetch", + "@types/phoenix", + "@types/ws@8.18.1", + "ws@8.18.3" + ] + }, + "@supabase/storage-js@2.12.1": { + "integrity": "sha512-QWg3HV6Db2J81VQx0PqLq0JDBn4Q8B1FYn1kYcbla8+d5WDmTdwwMr+EJAxNOSs9W4mhKMv+EYCpCrTFlTj4VQ==", + "dependencies": [ + "@supabase/node-fetch" ] }, "@supabase/storage-js@2.7.1": { @@ -98,6 +178,9 @@ "@supabase/node-fetch" ] }, + "@types/dom-mediacapture-record@1.0.22": { + "integrity": "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==" + }, "@types/node@22.13.0": { "integrity": "sha512-ClIbNe36lawluuvq3+YYhnIN2CELi+6q8NpnM7PYp4hBn/TatfboPgVSm2rwKRfnV2M+Ty9GWDFI64KEe+kysA==", "dependencies": [ @@ -107,6 +190,12 @@ "@types/phoenix@1.6.6": { "integrity": "sha512-PIzZZlEppgrpoT2QgbnDU+MMzuR6BbCjllj0bM70lWoejMeNJAxCchxnv7J3XFkI8MpygtRpzXrIlmWUBclP5A==" }, + "@types/ws@8.18.1": { + "integrity": "sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg==", + "dependencies": [ + "@types/node" + ] + }, "@types/ws@8.5.14": { "integrity": "sha512-bd/YFLW+URhBzMXurx7lWByOu+xzU9+kb3RboOteXYDfW+tr+JZa99OyNmPINEGB/ahzKrEuc8rcv4gnpJmxTw==", "dependencies": [ @@ -249,6 +338,9 @@ "etag@1.8.1": { "integrity": "sha512-aIL5Fx7mawVa300al2BnEE4iNvo1qETxLrPI/o05L7z6go7fCw1J6EQmbK4FmJ2AS7kgVF/KEZWufBfdClMcPg==" }, + "events@3.3.0": { + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==" + }, "eventsource-parser@3.0.2": { "integrity": "sha512-6RxOBZ/cYgd8usLwsEl+EC09Au/9BcmCKYF2/xbml6DNczf7nv0MQb+7BA2F+li6//I+28VNlQR37XfQtcAJuA==" }, @@ -460,6 +552,24 @@ "safe-buffer" ] }, + "livekit-client@2.15.7_@types+dom-mediacapture-record@1.0.22": { + "integrity": "sha512-19m8Q1cvRl5PslRawDUgWXeP8vL8584tX8kiZEJaPZo83U/L6VPS/O7pP06phfJaBWeeV8sAOVtEPlQiZEHtpg==", + "dependencies": [ + "@livekit/mutex", + "@livekit/protocol", + "@types/dom-mediacapture-record", + "events", + "loglevel", + "sdp-transform", + "ts-debounce", + "tslib", + "typed-emitter", + "webrtc-adapter" + ] + }, + "loglevel@1.9.2": { + "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==" + }, "math-intrinsics@1.1.0": { "integrity": "sha512-/IXtbwEk5HTPyEwyKX6hGkYXxM9nbj64B+ilVJnC/R6B0pH5G4V3b0pVbL7DBj4tkhBAppbQUlf6F6Xl9LHu1g==" }, @@ -558,12 +668,25 @@ "path-to-regexp" ] }, + "rxjs@7.8.2": { + "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", + "dependencies": [ + "tslib" + ] + }, "safe-buffer@5.2.1": { "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, "safer-buffer@2.1.2": { "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" }, + "sdp-transform@2.15.0": { + "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==", + "bin": true + }, + "sdp@3.2.1": { + "integrity": "sha512-lwsAIzOPlH8/7IIjjz3K0zYBk7aBVVcvjMwt3M4fLxpjMYyy7i3I97SLHebgn4YBjirkzfp3RvRDWSKsh/+WFw==" + }, "send@1.2.0": { "integrity": "sha512-uaW0WwXKpL9blXE2o0bRhoL2EGXIrZxQ2ZQ4mgcfoBxdFmQold+qWsD2jLrfZ0trjKL6vOw0j//eAwcALFjKSw==", "dependencies": [ @@ -646,6 +769,12 @@ "tr46@0.0.3": { "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==" }, + "ts-debounce@4.0.0": { + "integrity": "sha512-+1iDGY6NmOGidq7i7xZGA4cm8DAa6fqdYcvO5Z6yBevH++Bdo9Qt/mN0TzHUgcCcKv1gmh9+W5dHqz8pMWbCbg==" + }, + "tslib@2.8.1": { + "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==" + }, "type-is@2.0.1": { "integrity": "sha512-OZs6gsjF4vMp32qrCbiVSkrFmXtG/AZhY3t0iAMrMBiAZyV9oALtXO8hsrHbMXF9x6L3grlFuwW2oAz7cav+Gw==", "dependencies": [ @@ -654,6 +783,12 @@ "mime-types" ] }, + "typed-emitter@2.1.0": { + "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", + "optionalDependencies": [ + "rxjs" + ] + }, "undici-types@6.20.0": { "integrity": "sha512-Ny6QZ2Nju20vw1SRHe3d9jVu6gJ+4e3+MMpqu7pqE5HT6WsTSlce++GQmK5UXS8mzV8DSYHrQH+Xrf2jVcuKNg==" }, @@ -680,6 +815,12 @@ "webidl-conversions@3.0.1": { "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==" }, + "webrtc-adapter@9.0.3": { + "integrity": "sha512-5fALBcroIl31OeXAdd1YUntxiZl1eHlZZWzNg3U4Fn+J9/cGL3eT80YlrsWGvj2ojuz1rZr2OXkgCzIxAZ7vRQ==", + "dependencies": [ + "sdp" + ] + }, "whatwg-url@5.0.0": { "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==", "dependencies": [ @@ -700,6 +841,9 @@ "ws@8.18.0": { "integrity": "sha512-8VbfWfHLbbwu3+N6OKsOMpBdT4kXPDDB9cJk2bJ6mh9ucxdlnNvH1e+roYkKmN9Nxw2yjz7VzeO9oOz2zJ04Pw==" }, + "ws@8.18.3": { + "integrity": "sha512-PEIGCY5tSlUt50cqyMXfCzX+oOPqN0vuGqWzbcJ2xvnkzkq46oOpz7dQaTDBdfICb4N14+GARUDw2XV2N4tvzg==" + }, "zod-to-json-schema@3.24.5_zod@3.25.36": { "integrity": "sha512-/AuWwMP+YqiPbsJx5D6TfgRTc4kTLjsh5SOcd4bLsfUg2RcEXrFMJl1DGgdHy2aCfsIA/cr/1JM0xcB2GZji8g==", "dependencies": [ @@ -796,6 +940,7 @@ "dependencies": [ "jsr:@std/assert@1", "jsr:@supabase/supabase-js@^2.48.1", + "npm:@elevenlabs/client@~0.6.2", "npm:@evan/opus@^1.0.3" ] } diff --git a/server-deno/logs.log b/server-deno/logs.log new file mode 100644 index 0000000..7399242 --- /dev/null +++ b/server-deno/logs.log @@ -0,0 +1 @@ +Audio capture server running on ws://0.0.0.0:8000 diff --git a/server-deno/main.ts b/server-deno/main.ts index dab0dc5..fb83aca 100644 --- a/server-deno/main.ts +++ b/server-deno/main.ts @@ -4,7 +4,7 @@ import type { WebSocket as WSWebSocket, WebSocketServer as _WebSocketServer, } from "npm:@types/ws"; -import { authenticateUser } from "./utils.ts"; +import { authenticateUser, elevenLabsApiKey } from "./utils.ts"; import { createFirstMessage, createSystemPrompt, @@ -15,6 +15,7 @@ import { SupabaseClient } from "@supabase/supabase-js"; import { isDev } from "./utils.ts"; import { connectToOpenAI } from "./models/openai.ts"; import { connectToGemini } from "./models/gemini.ts"; +import { connectToElevenLabs } from "./models/elevenlabs.ts"; const server = createServer(); @@ -75,6 +76,21 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { systemPrompt, ); break; + case "elevenlabs": + const agentId = user.personality?.oai_voice ?? ""; + + if (!elevenLabsApiKey) { + throw new Error("ELEVENLABS_API_KEY environment variable is required"); + } + + await connectToElevenLabs( + ws, + payload, + connectionPcmFile, + agentId, + elevenLabsApiKey, + ); + break; default: throw new Error(`Unknown provider: ${provider}`); } diff --git a/server-deno/models/elevenlabs.ts b/server-deno/models/elevenlabs.ts new file mode 100644 index 0000000..6d9af8c --- /dev/null +++ b/server-deno/models/elevenlabs.ts @@ -0,0 +1,348 @@ +import { Buffer } from "node:buffer"; +import type { RawData } from "npm:@types/ws"; +// @ts-ignore +import { + WebSocketConnection, + type SessionConfig, + type IncomingSocketEvent, + type DisconnectionDetails +} from "npm:@elevenlabs/client"; + +import { addConversation, getDeviceInfo } from "../supabase.ts"; +import { encoder, FRAME_SIZE, isDev } from "../utils.ts"; +import type { IPayload } from "../types.d.ts"; + +// Calculate audio level for debugging +function calculateAudioLevel(audioData: any): number { + if (!audioData || audioData.length === 0) return 0; + + // Convert to 16-bit samples + const samples = new Int16Array(audioData.buffer || audioData); + let sum = 0; + + for (let i = 0; i < samples.length; i++) { + sum += Math.abs(samples[i]); + } + + return Math.round(sum / samples.length); +} + +export const connectToElevenLabs = async ( + ws: WebSocket, + payload: IPayload, + connectionPcmFile: Deno.FsFile | null, + agentId: string, + apiKey: string, +) => { + console.log(apiKey, agentId); + const { user, supabase } = payload; + + // Queue messages until ElevenLabs connection is ready + const messageQueue: RawData[] = []; + let isElevenLabsConnected = false; + let elevenLabsConnection: WebSocketConnection | null = null; + let hasResponseStarted = false; + + // Handle messages from ESP32 client + const handleClientMessage = async (data: any, isBinary: boolean) => { + try { + if (isBinary) { + const base64Data = data.toString("base64"); + + if (isDev && connectionPcmFile) { + await connectionPcmFile.write(data); + } + + // Send audio to ElevenLabs using their client + if (isElevenLabsConnected && elevenLabsConnection) { + // Check if audio contains actual speech (simple volume check) + const audioLevel = calculateAudioLevel(data); + console.log(`Sending audio chunk to ElevenLabs: raw=${data.length} bytes, base64=${base64Data.length} chars, level=${audioLevel}`); + + try { + elevenLabsConnection.sendMessage({ + user_audio_chunk: base64Data, + }); + } catch (error) { + console.error("Error sending audio to ElevenLabs:", error); + } + } else { + console.log(`Cannot send audio - ElevenLabs connected: ${isElevenLabsConnected}, connection exists: ${!!elevenLabsConnection}`); + } + } else { + const message = JSON.parse(data.toString("utf-8")); + + if (message.type === "instruction") { + switch (message.msg) { + case "INTERRUPT": + console.log("Interrupt detected"); + if (elevenLabsConnection) { + elevenLabsConnection.sendMessage({ + type: "user_activity" + }); + } + break; + + case "END_SESSION": + console.log("End session requested"); + if (elevenLabsConnection) { + elevenLabsConnection.close(); + } + break; + } + } + } + } catch (error) { + console.error("Error handling client message:", error); + } + }; + + try { + // For server-side usage, we need to get a signed URL first + const signedUrlResponse = await fetch( + `https://api.elevenlabs.io/v1/convai/conversation/get-signed-url?agent_id=${agentId}`, + { + headers: { + 'xi-api-key': apiKey, + }, + } + ); + + if (!signedUrlResponse.ok) { + throw new Error(`Failed to get signed URL: ${signedUrlResponse.status} ${signedUrlResponse.statusText}`); + } + + const { signed_url } = await signedUrlResponse.json(); + + // Use default audio formats (let ElevenLabs auto-detect) + const modifiedSignedUrl = signed_url; + + // Create ElevenLabs connection using signed URL for server-side usage + const sessionConfig: SessionConfig = { + signedUrl: modifiedSignedUrl, + connectionType: "websocket", + }; + + elevenLabsConnection = await WebSocketConnection.create(sessionConfig); + + console.log("Connected to ElevenLabs successfully!"); + isElevenLabsConnected = true; + console.log(`ElevenLabs connection ready - conversation_initiation_metadata already processed by SDK`); + + // Send initial RESPONSE.CREATED for the first message + console.log("Sending initial RESPONSE.CREATED to ESP32"); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.CREATED" + })); + + // Set up ElevenLabs event handlers + elevenLabsConnection.onMessage(async (event: IncomingSocketEvent) => { + console.log("ElevenLabs message type:", event.type); + + switch (event.type) { + case "conversation_initiation_metadata": + console.log("ElevenLabs conversation initiated (metadata received)"); + // RESPONSE.CREATED already sent when connection was established + break; + + case "ping": + // Handle ping messages - send pong response + console.log("Received ping from ElevenLabs, sending pong"); + if (event.ping_event?.event_id) { + elevenLabsConnection.sendMessage({ + type: "pong", + event_id: event.ping_event.event_id + }); + } + break; + + case "audio": + if (event.audio_event?.audio_base_64) { + // Send RESPONSE.CREATED only for the first audio chunk of each response + if (!hasResponseStarted) { + console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)"); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.CREATED" + })); + hasResponseStarted = true; + } + + const audioBuffer = Buffer.from(event.audio_event.audio_base_64, "base64"); + console.log(`Received audio from ElevenLabs: ${audioBuffer.length} bytes, processing into ${Math.ceil(audioBuffer.length / FRAME_SIZE)} frames`); + + let framesSent = 0; + // Process audio in frames for Opus encoding + for (let offset = 0; offset < audioBuffer.length; offset += FRAME_SIZE) { + const frame = audioBuffer.subarray(offset, offset + FRAME_SIZE); + + try { + const encodedPacket = encoder.encode(frame); + ws.send(encodedPacket); + framesSent++; + } catch (_e) { + // Skip this frame but continue with others + console.log(`Failed to encode frame at offset ${offset}`); + } + } + console.log(`Sent ${framesSent} audio frames to ESP32`); + } + break; + + case "user_transcript": + if (event.user_transcription_event?.user_transcript) { + console.log("User transcript:", event.user_transcription_event.user_transcript); + addConversation( + supabase, + "user", + event.user_transcription_event.user_transcript, + user, + ); + + // Send audio committed message like OpenAI does + console.log("Sending AUDIO.COMMITTED to ESP32"); + ws.send(JSON.stringify({ + type: "server", + msg: "AUDIO.COMMITTED" + })); + + + } + break; + + case "agent_response": + if (event.agent_response_event?.agent_response) { + console.log("Agent response:", event.agent_response_event.agent_response); + addConversation( + supabase, + "assistant", + event.agent_response_event.agent_response, + user, + ); + + // Send response complete with device info like OpenAI does + console.log("Sending RESPONSE.COMPLETE to ESP32"); + hasResponseStarted = false; // Reset for next response + try { + const device = await getDeviceInfo(supabase, user.user_id); + + if (device) { + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.COMPLETE", + volume_control: device.volume ?? 100, + })); + } else { + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.COMPLETE", + })); + } + } catch (error) { + console.error("Error fetching updated device info:", error); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.COMPLETE", + })); + } + } + break; + + case "vad_score": + // Voice Activity Detection score - can be used for debugging + if (event.vad_score_event?.vad_score) { + console.log("VAD score:", event.vad_score_event.vad_score); + } + break; + + case "internal_tentative_agent_response": + // Tentative response while agent is thinking + if (event.tentative_agent_response_internal_event?.tentative_agent_response) { + console.log("Tentative response:", event.tentative_agent_response_internal_event.tentative_agent_response); + } + break; + + case "conversation_end": + console.log("ElevenLabs conversation ended"); + ws.send(JSON.stringify({ + type: "server", + msg: "SESSION.END" + })); + break; + + default: + console.log("Unknown ElevenLabs message:", event.type, event); + } + }); + + elevenLabsConnection.onDisconnect((details: DisconnectionDetails) => { + console.log("ElevenLabs connection closed:", details.reason); + ws.close(); + }); + + // Process queued messages + while (messageQueue.length > 0) { + const queuedMessage = messageQueue.shift(); + if (queuedMessage) { + handleClientMessage(queuedMessage, false); + } + } + + // Set up ESP32 WebSocket handlers + ws.on("message", (data: any, isBinary: boolean) => { + if (!isElevenLabsConnected) { + messageQueue.push(data); + } else { + handleClientMessage(data, isBinary); + } + }); + + ws.on("error", (error: any) => { + console.error("ESP32 WebSocket error:", error); + elevenLabsConnection?.close(); + }); + + ws.on("close", async (code: number, reason: string) => { + console.log(`ESP32 WebSocket closed with code ${code}, reason: ${reason}`); + elevenLabsConnection?.close(); + + if (isDev && connectionPcmFile) { + connectionPcmFile.close(); + console.log("Closed debug audio file."); + } + }); + + // Send initial device info to ESP32 + try { + const device = await getDeviceInfo(supabase, user.user_id); + + if (device) { + ws.send(JSON.stringify({ + type: "server", + msg: "DEVICE.INFO", + volume_control: device.volume ?? 100, + })); + } + } catch (error) { + console.error("Error fetching device info:", error); + } + + } catch (error) { + console.error("Failed to connect to ElevenLabs:", error); + + // Send more specific error information + let errorMessage = "RESPONSE.ERROR"; + if (error instanceof Error) { + console.error("Error details:", error.message); + if (error.message.includes("signed URL")) { + errorMessage = "AUTH.ERROR"; + } + } + + ws.send(JSON.stringify({ + type: "server", + msg: errorMessage + })); + } +}; \ No newline at end of file diff --git a/server-deno/types.d.ts b/server-deno/types.d.ts index d30c94d..874e2f8 100644 --- a/server-deno/types.d.ts +++ b/server-deno/types.d.ts @@ -26,7 +26,7 @@ declare global { user_code: string; } - type ModelProvider = "openai" | "gemini"; + type ModelProvider = "openai" | "gemini" | "elevenlabs"; type GeminiVoice = | "Zephyr" @@ -70,13 +70,17 @@ declare global { | "sage" | "verse"; + /** + * Note: oai_voice is essentially the name of the voice. + * the naming here sucks, please change it + */ interface IPersonality { personality_id: string; is_doctor: boolean; is_child_voice: boolean; is_story: boolean; key: string; - oai_voice: OaiVoice | GeminiVoice; + oai_voice: string; provider: ModelProvider; voice_description: string; title: string; diff --git a/server-deno/utils.ts b/server-deno/utils.ts index d8a0fa1..be46e07 100644 --- a/server-deno/utils.ts +++ b/server-deno/utils.ts @@ -26,6 +26,7 @@ encoder.bitrate = 12000; export const openaiApiKey = Deno.env.get("OPENAI_API_KEY"); export const geminiApiKey = Deno.env.get("GEMINI_API_KEY"); +export const elevenLabsApiKey = Deno.env.get("ELEVENLABS_API_KEY"); export { encoder, FRAME_SIZE };