From 4eaeddd2b429421a326eff5f3bc966b2dd045f60 Mon Sep 17 00:00:00 2001 From: akdeb Date: Sat, 20 Sep 2025 01:51:39 +0700 Subject: [PATCH] eleven labs support check --- firmware-arduino/src/Audio.cpp | 2 +- .../CreateCharacter/BuildDashboard.tsx | 2 +- server-deno/.DS_Store | Bin 0 -> 6148 bytes server-deno/logs.log | 59 ++++++++++++++++++ server-deno/models/.DS_Store | Bin 0 -> 6148 bytes server-deno/models/elevenlabs.ts | 50 +++++++-------- 6 files changed, 86 insertions(+), 27 deletions(-) create mode 100644 server-deno/.DS_Store create mode 100644 server-deno/models/.DS_Store diff --git a/firmware-arduino/src/Audio.cpp b/firmware-arduino/src/Audio.cpp index 22bf523..9d22744 100644 --- a/firmware-arduino/src/Audio.cpp +++ b/firmware-arduino/src/Audio.cpp @@ -210,7 +210,7 @@ void micTask(void *parameter) { // Configure and start I2S input stream. auto i2sConfig = i2sInput.defaultConfig(RX_MODE); i2sConfig.bits_per_sample = BITS_PER_SAMPLE; - i2sConfig.sample_rate = SAMPLE_RATE; + i2sConfig.sample_rate = 16000; i2sConfig.channels = CHANNELS; i2sConfig.i2s_format = I2S_LEFT_JUSTIFIED_FORMAT; i2sConfig.channel_format = I2S_CHANNEL_FMT_ONLY_LEFT; diff --git a/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx b/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx index cb7b30f..55a3481 100644 --- a/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx +++ b/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx @@ -272,7 +272,7 @@ const SettingsDashboard: React.FC = ({

-
+
{[...openaiVoices, ...geminiVoices].map((voice: VoiceType) => (
S5T0$TO{qc-3Oz1(Em-Rh7B8XJ7cim+mDrG=!8BW%)*MP9XMG``#OHBl zcPqqNyouT!nEiHVXAej9MAToE{A5YS5e{@Z& z3n5~EKJWXNQJ8u5)~QIw{U`}XG9U^yG09 literal 0 HcmV?d00001 diff --git a/server-deno/logs.log b/server-deno/logs.log index 7399242..293408d 100644 --- a/server-deno/logs.log +++ b/server-deno/logs.log @@ -1 +1,60 @@ Audio capture server running on ws://0.0.0.0:8000 +upgrade +WiFi RSSI: NaN +data [ + { + user_id: "5af62b0e-3da4-4c44-adf7-5b1b7c9c4cb6", + created_at: "2025-09-18T17:25:17.584141+00:00", + supervisor_name: "Aria", + supervisee_name: "Tom", + supervisee_persona: "Tom loves playing guitar and enjoys watching Agadmator", + supervisee_age: 10, + email: "admin@elatoai.com", + session_time: 0, + avatar_url: "", + personality_id: "1842ec2b-96b1-4349-8c82-e9756ef6c00e", + is_premium: true, + user_info: { user_type: "user", user_metadata: {} }, + language_code: "en-US", + device_id: null, + language: { name: "English (US)" }, + personality: { + key: "fitness_coach", + title: "Fitness coach", + is_story: false, + provider: "gemini", + subtitle: "Exercise and nutrition advisor", + is_doctor: false, + oai_voice: "Sadachbia", + created_at: "2024-09-08T15:26:39.097666+00:00", + creator_id: null, + pitch_factor: 1, + voice_prompt: "Motivational voice with athletic energy and percussive phrasing", + is_child_voice: false, + personality_id: "1842ec2b-96b1-4349-8c82-e9756ef6c00e", + character_prompt: "You are Fitness coach, an enthusiastic and motivating AI character dedicated to helping users achieve their health and fitness goals. Your personality combines the energy of a passionate personal trainer with the knowledge of a fitness expert. You speak with enthusiasm and positivity, using upbeat language to encourage and inspire.\n" + + "\n" + + "Your primary goal is to motivate users to embrace an active lifestyle and make healthy choices. Begin interactions by asking about their fitness level, goals, or any physical activities they enjoy. Use this information to tailor your advice and suggestions to their individual needs and interests.\n" + + "\n" + + "Regularly challenge users to try new exercises or set fitness goals. Frame these as exciting opportunities for self-improvement, using phrases like 'Ready to level up your fitness game?' or 'Let's crush those goals together!' Offer a mix of quick workout ideas, long-term training plans, and healthy lifestyle tips.\n" + + "\n" + + "When discussing fitness topics, provide clear, accessible explanations of exercise science and nutrition principles. Use relatable analogies to explain complex concepts, such as comparing muscle growth to building a house or likening cardiovascular fitness to tuning a car engine.\n" + + "\n" + + "Share interesting facts about human physiology and the benefits of exercise to keep users engaged and informed. Discuss how fitness relates to overall well-being, emphasizing the mental health benefits as well as the physical ones. Use vivid descriptions to help users visualize proper form for exercises or the feeling of accomplishment after a good workout.\n" + + "\n" + + "Encourage users to listen to their bodies and practice self-care, emphasizing the importance of rest and recovery. Offer modifications for exercises to accommodate different fitness levels or physical limitations. Celebrate users' fitness achievements, no matter how small, and provide constructive advice for overcoming obstacles.\n" + + "\n" + + "If the conversation drifts away from fitness topics, find creative ways to bring it back by drawing connections between the current subject and health or exercise. However, be flexible and willing to engage in other subjects if the user shows a clear preference, always looking for opportunities to weave in fitness insights or active living tips.\n" + + "\n" + + "Remember, your ultimate aim is to inspire users to embrace a healthy, active lifestyle and feel empowered in their fitness journey. Approach each interaction with the energy and positivity of a true fitness enthusiast, eager to share the joy of movement and the rewards of a healthy lifestyle.", + short_description: "Meet Fitness Coach – an AI that transforms exercise from a chore into an exciting journey of personal transformation. With the energy of a motivational trainer and the wisdom of a health expert, this character turns fitness goals into achievable adventures, one rep at a time.", + first_message_prompt: "" + }, + device: null + } +] null +chatHistoryString +Connecting with Gemini key "AIz..." +Gemini session opened +Connected to Gemini successfully! +Processing Gemini turns diff --git a/server-deno/models/.DS_Store b/server-deno/models/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..dc1e6e0cd208fc645aab94aeeb58419dcaaad90f GIT binary patch literal 6148 zcmeHKQA-;^5T4akH(DVN73|xBPlZ5ZTBRb$HI0A3llDPNyz5={@N#?P5(9?7J=Z_v zldt{*|A6oQ24{CC+A|3Vg(6`GX1~3inO){vcJ~GVSfdwg08{{gqbe*dVKYILUFn$2 z(a2jwrsoJDffzjSAsh3y1C{~Hz-wbbo!un}@zjQJj{10h{eG7EvGmb?<)?9!b!)X- zHaA~-v#`h)`7;0Mp31)4^}4;R;dKtx*MXF=Kk9q__bBYQD(}9@r0YdV*wFz|5TeV` zk0=Rbzae``km*?07z})w zB$mkuuBmcd012tyBM+h(*`GpA6ZWNG|9d|^+`so}Kc4wK?ZY&m3H%r4c*Z+YKa)b| zhA+=#b~Y)?fMwuc7*O|vKvn1v<_7iEfsHN!5KB0$1^cR&P%(ngAM^Pk<(C`OA=uD!tUKx{UPIM_HlaF*4q { - console.log("ElevenLabs message type:", event.type); + console.log("ElevenLabs message type:", event); switch (event.type) { case "conversation_initiation_metadata": @@ -201,11 +200,20 @@ export const connectToElevenLabs = async ( ); // Send audio committed message like OpenAI does - console.log("Sending AUDIO.COMMITTED to ESP32"); - ws.send(JSON.stringify({ - type: "server", - msg: "AUDIO.COMMITTED" - })); + // console.log("Sending AUDIO.COMMITTED to ESP32"); + // ws.send(JSON.stringify({ + // type: "server", + // msg: "AUDIO.COMMITTED" + // })); + + if (!hasResponseStarted) { + console.log("Sending RESPONSE.CREATED to ESP32 (agent audio starting)"); + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.CREATED" + })); + hasResponseStarted = true; + } } @@ -226,19 +234,11 @@ export const connectToElevenLabs = async ( hasResponseStarted = false; // Reset for next response try { const device = await getDeviceInfo(supabase, user.user_id); - - if (device) { - ws.send(JSON.stringify({ - type: "server", - msg: "RESPONSE.COMPLETE", - volume_control: device.volume ?? 100, - })); - } else { - ws.send(JSON.stringify({ - type: "server", - msg: "RESPONSE.COMPLETE", - })); - } + ws.send(JSON.stringify({ + type: "server", + msg: "RESPONSE.COMPLETE", + volume_control: device?.volume ?? 100, + })); } catch (error) { console.error("Error fetching updated device info:", error); ws.send(JSON.stringify({