admin-valentin/AIvoices
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

adding tool calling

Browse source
This commit is contained in:
akdeb 2025-06-09 18:55:19 +01:00
parent 646a46b911
commit 77e6adf4bf
5 changed files with 75 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
firmware-arduino/src/Audio.cpp
View file

@ -332,7 +332,10 @@ void webSocketEvent(WStype_t type, uint8_t *payload, size_t length)
} else if (strcmp((char*)msg.c_str(), "RESPONSE.CREATED") == 0) {
Serial.println("Received RESPONSE.CREATED, transitioning to speaking");
transitionToSpeaking();
}
} else if (strcmp((char*)msg.c_str(), "SESSION.END") == 0) {
Serial.println("Received SESSION.END, going to sleep");
sleepRequested = true;
}
}
}
break;

1
firmware-arduino/src/Config.cpp
View file

@ -5,6 +5,7 @@
Preferences preferences;
OtaStatus otaState = OTA_IDLE;
bool factory_reset_status = false;
volatile bool sleepRequested = false;
/**
* Configuration for Elato Firmware

3
firmware-arduino/src/Config.h
View file

@ -92,9 +92,12 @@ extern const int I2S_DATA_OUT;
extern const i2s_port_t I2S_PORT_OUT;
extern const int I2S_SD_OUT;
extern volatile bool sleepRequested;
// SSL certificate
extern const char *CA_cert;
extern const char *Vercel_CA_cert;
void factoryResetDevice();
void processSleepRequest();
#endif

17
firmware-arduino/src/main.cpp
View file

@ -10,9 +10,8 @@
#include "FactoryReset.h"
#define TOUCH_THRESHOLD 28000
#define LONG_PRESS_MS 1000
#define REQUIRED_RELEASE_CHECKS 100 // how many consecutive times we need "below threshold" to confirm release
#define TOUCH_DEBOUNCE_DELAY 1000 // milliseconds
#define TOUCH_DEBOUNCE_DELAY 500 // milliseconds
AsyncWebServer webServer(80);
WIFIMANAGER WifiManager;
@ -67,6 +66,13 @@ void enterSleep()
delay(1000);
}
void processSleepRequest() {
if (sleepRequested) {
sleepRequested = false;
enterSleep(); // Just call it directly - no state checking needed
}
}
void printOutESP32Error(esp_err_t err)
{
switch (err)
@ -90,14 +96,14 @@ static void onButtonLongPressUpEventCb(void *button_handle, void *usr_data)
{
Serial.println("Button long press end");
delay(10);
enterSleep();
sleepRequested = true;
}
static void onButtonDoubleClickCb(void *button_handle, void *usr_data)
{
Serial.println("Button double click");
delay(10);
enterSleep();
sleepRequested = true;
}
void getAuthTokenFromNVS()
@ -149,7 +155,7 @@ void touchTask(void* parameter) {
// Check for long press while touched
if (touched && isTouched) {
if (currentTime - pressStartTime >= LONG_PRESS_DURATION) {
enterSleep(); // Only enter sleep after 500ms of continuous touch
sleepRequested = true; // Only enter sleep after 500ms of continuous touch
}
}
@ -247,6 +253,7 @@ void setup()
}
void loop(){
processSleepRequest();
if (otaState == OTA_IN_PROGRESS)
{
loopOTA();

77
server-deno/main.ts
View file

@ -21,21 +21,6 @@ import {
} from "./supabase.ts";
import { SupabaseClient } from "@supabase/supabase-js";
// import { Opus, OpusApplication } from 'https://deno.land/x/opus@0.1.1/opus.ts';
// await Opus.load(); // Make sure the Opus module is loaded
// // Define your audio parameters (adjust as needed)
// const SAMPLE_RATE = 24000; // e.g., 48000 Hz
// const CHANNELS = 1; // Stereo (change to 1 if using mono)
// const FRAME_DURATION = 20; // Frame length in ms
// // Calculate the number of samples per frame
// const frameSize = (SAMPLE_RATE * FRAME_DURATION) / 1000;
// // Create a global encoder instance (reuse this for every audio delta)
// const encoder = new Opus(SAMPLE_RATE, CHANNELS, OpusApplication.AUDIO);
import { Encoder } from "@evan/opus";
const isDev = Deno.env.get("DEV_MODE") === "True";
@ -46,9 +31,6 @@ const CHANNELS = 1; // Mono (set to 2 if you have stereo)
const FRAME_DURATION = 120; // Frame length in ms
const BYTES_PER_SAMPLE = 2; // 16-bit PCM: 2 bytes per sample
// Calculate the number of bytes per frame:
// samples = SAMPLE_RATE * FRAME_DURATION / 1000
// bytes = samples * CHANNELS * BYTES_PER_SAMPLE
const FRAME_SIZE = (SAMPLE_RATE * FRAME_DURATION / 1000) * CHANNELS *
BYTES_PER_SAMPLE; // 960 bytes for 24000 Hz mono 16-bit
@ -115,7 +97,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
ws.send(
JSON.stringify({
type: "auth",
volume_control: user.device?.volume ?? 100,
volume_control: user.device?.volume ?? 20,
is_ota: user.device?.is_ota ?? false,
is_reset: user.device?.is_reset ?? false,
pitch_factor: user.personality?.pitch_factor ?? 1,
@ -135,11 +117,60 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
const systemPrompt = createSystemPrompt(chatHistory, payload);
let sessionStartTime: number;
let currentItemId: string | null = null;
let currentCallId: string | null = null;
// Instantiate new client
console.log(`Connecting with key "${openaiApiKey.slice(0, 3)}..."`);
const client = new RealtimeClient({ apiKey: openaiApiKey });
// ADD TOOL CALLS HERE
client.addTool(
{
type: "function",
name: "end_session",
description:
'Call this if the user says bye or needs to leave or suggests they want to end the session. (e.g. "I gotta to go", "I have to work", "I have to sleep", "I have to do something else")',
parameters: {
type: "object",
strict: true,
properties: {
reason: {
type: "string",
description:
'Short reason for ending the session.',
},
},
required: ["reason"],
},
},
(args: any) => {
console.log("end session", args);
// Send your custom message to the client
ws.send(JSON.stringify({ type: "server", msg: "SESSION.END" }));
// Send the function result back to OpenAI
const functionResult = {
event_id: RealtimeUtils.generateId("evt_"),
type: "conversation.item.create",
item: {
id: RealtimeUtils.generateId("item_"),
type: "function_call_output",
call_id: currentCallId,
output: JSON.stringify({
success: true,
message: `Session ended: ${args.reason}`,
}),
},
};
client.realtime.send(functionResult.type, functionResult);
// Return the result for the callback
return { success: true, message: `Session ended: ${args.reason}` };
},
);
// Relay: OpenAI Realtime API Event -> Browser Event
client.realtime.on("server.*", async (event: any) => {
// console.log(`Relaying "${event.type}" to Client`);
@ -196,8 +227,6 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
if (event.type in client.conversation.EventProcessors) {
try {
const { delta } = client.conversation.processEvent(event);
switch (event.type) {
case "response.created":
console.log("response.created", event);
@ -213,10 +242,14 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
if (event.item.id) {
console.log("foobar", event.item.id);
currentItemId = event.item.id;
currentCallId = event.item.call_id;
}
break;
case "response.audio.delta":
{
const { delta } = client.conversation.processEvent(
event,
);
try {
if (delta?.audio?.buffer) {
const pcmBuffer = Buffer.from(
@ -401,7 +434,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
instructions: systemPrompt,
input_audio_transcription: { model: "whisper-1" },
};
await client.connect(sessionOptions);
await client.connect(sessionOptions as any);
} catch (e: unknown) {
console.log(`Error connecting to OpenAI: ${e as Error}`);
ws.close();