diff --git a/firmware-arduino/src/Audio.cpp b/firmware-arduino/src/Audio.cpp index 12d64be..8774d64 100644 --- a/firmware-arduino/src/Audio.cpp +++ b/firmware-arduino/src/Audio.cpp @@ -332,7 +332,10 @@ void webSocketEvent(WStype_t type, uint8_t *payload, size_t length) } else if (strcmp((char*)msg.c_str(), "RESPONSE.CREATED") == 0) { Serial.println("Received RESPONSE.CREATED, transitioning to speaking"); transitionToSpeaking(); - } + } else if (strcmp((char*)msg.c_str(), "SESSION.END") == 0) { + Serial.println("Received SESSION.END, going to sleep"); + sleepRequested = true; + } } } break; diff --git a/firmware-arduino/src/Config.cpp b/firmware-arduino/src/Config.cpp index f8f60b7..67b372d 100644 --- a/firmware-arduino/src/Config.cpp +++ b/firmware-arduino/src/Config.cpp @@ -5,6 +5,7 @@ Preferences preferences; OtaStatus otaState = OTA_IDLE; bool factory_reset_status = false; +volatile bool sleepRequested = false; /** * Configuration for Elato Firmware diff --git a/firmware-arduino/src/Config.h b/firmware-arduino/src/Config.h index e4980e5..c49ef3d 100644 --- a/firmware-arduino/src/Config.h +++ b/firmware-arduino/src/Config.h @@ -92,9 +92,12 @@ extern const int I2S_DATA_OUT; extern const i2s_port_t I2S_PORT_OUT; extern const int I2S_SD_OUT; +extern volatile bool sleepRequested; + // SSL certificate extern const char *CA_cert; extern const char *Vercel_CA_cert; void factoryResetDevice(); +void processSleepRequest(); #endif \ No newline at end of file diff --git a/firmware-arduino/src/main.cpp b/firmware-arduino/src/main.cpp index f33c024..f29721c 100644 --- a/firmware-arduino/src/main.cpp +++ b/firmware-arduino/src/main.cpp @@ -10,9 +10,8 @@ #include "FactoryReset.h" #define TOUCH_THRESHOLD 28000 -#define LONG_PRESS_MS 1000 #define REQUIRED_RELEASE_CHECKS 100 // how many consecutive times we need "below threshold" to confirm release -#define TOUCH_DEBOUNCE_DELAY 1000 // milliseconds +#define TOUCH_DEBOUNCE_DELAY 500 // milliseconds AsyncWebServer webServer(80); WIFIMANAGER WifiManager; @@ -67,6 +66,13 @@ void enterSleep() delay(1000); } +void processSleepRequest() { + if (sleepRequested) { + sleepRequested = false; + enterSleep(); // Just call it directly - no state checking needed + } +} + void printOutESP32Error(esp_err_t err) { switch (err) @@ -90,14 +96,14 @@ static void onButtonLongPressUpEventCb(void *button_handle, void *usr_data) { Serial.println("Button long press end"); delay(10); - enterSleep(); + sleepRequested = true; } static void onButtonDoubleClickCb(void *button_handle, void *usr_data) { Serial.println("Button double click"); delay(10); - enterSleep(); + sleepRequested = true; } void getAuthTokenFromNVS() @@ -149,7 +155,7 @@ void touchTask(void* parameter) { // Check for long press while touched if (touched && isTouched) { if (currentTime - pressStartTime >= LONG_PRESS_DURATION) { - enterSleep(); // Only enter sleep after 500ms of continuous touch + sleepRequested = true; // Only enter sleep after 500ms of continuous touch } } @@ -247,6 +253,7 @@ void setup() } void loop(){ + processSleepRequest(); if (otaState == OTA_IN_PROGRESS) { loopOTA(); diff --git a/server-deno/main.ts b/server-deno/main.ts index 47eac4b..d26264c 100644 --- a/server-deno/main.ts +++ b/server-deno/main.ts @@ -21,21 +21,6 @@ import { } from "./supabase.ts"; import { SupabaseClient } from "@supabase/supabase-js"; -// import { Opus, OpusApplication } from 'https://deno.land/x/opus@0.1.1/opus.ts'; - -// await Opus.load(); // Make sure the Opus module is loaded - -// // Define your audio parameters (adjust as needed) -// const SAMPLE_RATE = 24000; // e.g., 48000 Hz -// const CHANNELS = 1; // Stereo (change to 1 if using mono) -// const FRAME_DURATION = 20; // Frame length in ms - -// // Calculate the number of samples per frame -// const frameSize = (SAMPLE_RATE * FRAME_DURATION) / 1000; - -// // Create a global encoder instance (reuse this for every audio delta) -// const encoder = new Opus(SAMPLE_RATE, CHANNELS, OpusApplication.AUDIO); - import { Encoder } from "@evan/opus"; const isDev = Deno.env.get("DEV_MODE") === "True"; @@ -46,9 +31,6 @@ const CHANNELS = 1; // Mono (set to 2 if you have stereo) const FRAME_DURATION = 120; // Frame length in ms const BYTES_PER_SAMPLE = 2; // 16-bit PCM: 2 bytes per sample -// Calculate the number of bytes per frame: -// samples = SAMPLE_RATE * FRAME_DURATION / 1000 -// bytes = samples * CHANNELS * BYTES_PER_SAMPLE const FRAME_SIZE = (SAMPLE_RATE * FRAME_DURATION / 1000) * CHANNELS * BYTES_PER_SAMPLE; // 960 bytes for 24000 Hz mono 16-bit @@ -115,7 +97,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { ws.send( JSON.stringify({ type: "auth", - volume_control: user.device?.volume ?? 100, + volume_control: user.device?.volume ?? 20, is_ota: user.device?.is_ota ?? false, is_reset: user.device?.is_reset ?? false, pitch_factor: user.personality?.pitch_factor ?? 1, @@ -135,11 +117,60 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { const systemPrompt = createSystemPrompt(chatHistory, payload); let sessionStartTime: number; let currentItemId: string | null = null; + let currentCallId: string | null = null; // Instantiate new client console.log(`Connecting with key "${openaiApiKey.slice(0, 3)}..."`); const client = new RealtimeClient({ apiKey: openaiApiKey }); + // ADD TOOL CALLS HERE + client.addTool( + { + type: "function", + name: "end_session", + description: + 'Call this if the user says bye or needs to leave or suggests they want to end the session. (e.g. "I gotta to go", "I have to work", "I have to sleep", "I have to do something else")', + parameters: { + type: "object", + strict: true, + properties: { + reason: { + type: "string", + description: + 'Short reason for ending the session.', + }, + }, + required: ["reason"], + }, + }, + (args: any) => { + console.log("end session", args); + + // Send your custom message to the client + ws.send(JSON.stringify({ type: "server", msg: "SESSION.END" })); + + // Send the function result back to OpenAI + const functionResult = { + event_id: RealtimeUtils.generateId("evt_"), + type: "conversation.item.create", + item: { + id: RealtimeUtils.generateId("item_"), + type: "function_call_output", + call_id: currentCallId, + output: JSON.stringify({ + success: true, + message: `Session ended: ${args.reason}`, + }), + }, + }; + + client.realtime.send(functionResult.type, functionResult); + + // Return the result for the callback + return { success: true, message: `Session ended: ${args.reason}` }; + }, + ); + // Relay: OpenAI Realtime API Event -> Browser Event client.realtime.on("server.*", async (event: any) => { // console.log(`Relaying "${event.type}" to Client`); @@ -196,8 +227,6 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { if (event.type in client.conversation.EventProcessors) { try { - const { delta } = client.conversation.processEvent(event); - switch (event.type) { case "response.created": console.log("response.created", event); @@ -213,10 +242,14 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { if (event.item.id) { console.log("foobar", event.item.id); currentItemId = event.item.id; + currentCallId = event.item.call_id; } break; case "response.audio.delta": { + const { delta } = client.conversation.processEvent( + event, + ); try { if (delta?.audio?.buffer) { const pcmBuffer = Buffer.from( @@ -401,7 +434,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => { instructions: systemPrompt, input_audio_transcription: { model: "whisper-1" }, }; - await client.connect(sessionOptions); + await client.connect(sessionOptions as any); } catch (e: unknown) { console.log(`Error connecting to OpenAI: ${e as Error}`); ws.close();