diff --git a/README.md b/README.md index 81086d2..ce98bbb 100644 --- a/README.md +++ b/README.md @@ -57,10 +57,10 @@ git clone git@github.com:akdeb/ElatoAI.git 2. **Start Supabase** -Install [Supabase CLI](https://supabase.com/docs/guides/local-development/cli/getting-started) and set up your Local Supabase Backend. From the root directory, run: +Install [Supabase CLI](https://supabase.com/docs/guides/local-development/cli/getting-started) and set up your Local Supabase Backend. Make sure you have Docker Desktop set up. Then from the root directory, run: ```bash brew install supabase/tap/supabase -supabase start # Starts your local Supabase server with the default migrations and seed data. +supabase start # This starts your local Supabase server with the default migrations and seed data. ``` 3. **Set up your NextJS Frontend** diff --git a/firmware-arduino/src/OTA.cpp b/firmware-arduino/src/OTA.cpp index 0caaab6..577dc9a 100644 --- a/firmware-arduino/src/OTA.cpp +++ b/firmware-arduino/src/OTA.cpp @@ -21,9 +21,6 @@ const char *server_certificate = R"EOF( void markOTAUpdateComplete() { HTTPClient http; - WiFiClientSecure client; - client.setCACert(Vercel_CA_cert); // Using the existing server certificate - // Construct the JSON payload JsonDocument doc; doc["authToken"] = authTokenGlobal; @@ -33,9 +30,11 @@ void markOTAUpdateComplete() { // Initialize HTTPS connection with client #ifdef DEV_MODE - http.begin("http://" + String(backend_server) + ":" + String(backend_port) + "/api/ota_update_handler"); + http.begin("http://" + String(backend_server) + ":" + String(backend_port) + "/api/ota_update_handler"); #else - http.begin(client, "https://" + String(backend_server) + "/api/ota_update_handler"); + WiFiClientSecure client; + client.setCACert(Vercel_CA_cert); // Using the existing server certificate + http.begin(client, "https://" + String(backend_server) + "/api/ota_update_handler"); #endif http.addHeader("Content-Type", "application/json"); diff --git a/firmware-arduino/src/WifiManager.cpp b/firmware-arduino/src/WifiManager.cpp index 0b94668..04dd774 100644 --- a/firmware-arduino/src/WifiManager.cpp +++ b/firmware-arduino/src/WifiManager.cpp @@ -19,15 +19,15 @@ bool isDeviceRegistered() { } HTTPClient http; - WiFiClientSecure client; - client.setCACert(Vercel_CA_cert); #ifdef DEV_MODE - http.begin("http://" + String(backend_server) + ":" + String(backend_port) + - "/api/generate_auth_token?macAddress=" + WiFi.macAddress()); + http.begin("http://" + String(backend_server) + ":" + String(backend_port) + + "/api/generate_auth_token?macAddress=" + WiFi.macAddress()); #else - http.begin(client, "https://" + String(backend_server) + - "/api/generate_auth_token?macAddress=" + WiFi.macAddress()); + WiFiClientSecure client; + client.setCACert(Vercel_CA_cert); + http.begin(client, "https://" + String(backend_server) + + "/api/generate_auth_token?macAddress=" + WiFi.macAddress()); #endif http.setTimeout(10000); diff --git a/server-deno/deno.lock b/server-deno/deno.lock index 74a2d2e..fa9c5f2 100644 --- a/server-deno/deno.lock +++ b/server-deno/deno.lock @@ -1,8 +1,6 @@ { - "version": "4", + "version": "5", "specifiers": { - "jsr:@std/assert@1": "1.0.11", - "jsr:@std/internal@^1.0.5": "1.0.5", "jsr:@supabase/supabase-js@2": "2.48.1", "jsr:@supabase/supabase-js@^2.48.1": "2.48.1", "npm:@evan/opus@^1.0.3": "1.0.3", @@ -15,15 +13,6 @@ "npm:ws@*": "8.18.0" }, "jsr": { - "@std/assert@1.0.11": { - "integrity": "2461ef3c368fe88bc60e186e7744a93112f16fd110022e113a0849e94d1c83c1", - "dependencies": [ - "jsr:@std/internal" - ] - }, - "@std/internal@1.0.5": { - "integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba" - }, "@supabase/supabase-js@2.48.1": { "integrity": "747e85c2a546efcc9f84b343b08e10e3818d468ae214506b59dc65a36fd2d702", "dependencies": [ @@ -79,8 +68,8 @@ "@supabase/node-fetch" ] }, - "@types/node@22.12.0": { - "integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==", + "@types/node@22.13.0": { + "integrity": "sha512-ClIbNe36lawluuvq3+YYhnIN2CELi+6q8NpnM7PYp4hBn/TatfboPgVSm2rwKRfnV2M+Ty9GWDFI64KEe+kysA==", "dependencies": [ "undici-types" ] @@ -194,12 +183,7 @@ "https://deno.land/x/jose@v5.9.6/util/decode_jwt.ts": "a6347360591bd60b076d115fc231e3ecd84aef319922a6c6e94554e68863e3af", "https://deno.land/x/jose@v5.9.6/util/decode_protected_header.ts": "3412c8aacffee93148478f14a0ebb97b2ff060944f49dd233223d262f592ee36", "https://deno.land/x/jose@v5.9.6/util/errors.ts": "b7ecf22990ce73a42650fd09666fb7174fcd91eec387aa26b361269c8022220f", - "https://deno.land/x/jose@v5.9.6/util/runtime.ts": "088cae7df0d285065f141d880c5c21a572234997082337d1125d7f1603059642", - "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/api.js": "fb958c31e4c4e04dd4bee56c35f8cd0b69b72dd6abf297e3bf71a498f1f07f5a", - "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/client.js": "390bcc3d817f4c3deadcec4e65ce8e915e4f8ae19aee32d62de2e71f9adab65d", - "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/conversation.js": "f2ffef21963a669d705cfccdbc4ee92524e81f8680d0cb12914f18eebc2b1369", - "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/event_handler.js": "0eeea0fd7d288b25995233a644632b61c84d5af196ba5d0422d0b4040058c595", - "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/utils.js": "aecdf22b2530c1b5763a5888d43f3bdc76266f63ed0628c35ab54518e7dd436b" + "https://deno.land/x/jose@v5.9.6/util/runtime.ts": "088cae7df0d285065f141d880c5c21a572234997082337d1125d7f1603059642" }, "workspace": { "dependencies": [ diff --git a/server-deno/main.ts b/server-deno/main.ts index bc59cce..83a03d8 100644 --- a/server-deno/main.ts +++ b/server-deno/main.ts @@ -7,8 +7,8 @@ import type { WebSocketServer as _WebSocketServer, } from "npm:@types/ws"; -import { RealtimeClient } from "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/client.js"; -import { RealtimeUtils } from "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/utils.js"; +import { RealtimeClient } from "./realtime/client.js"; +import { RealtimeUtils } from "./realtime/utils.js"; import { authenticateUser } from "./utils.ts"; import { addConversation, diff --git a/server-deno/realtime/api.js b/server-deno/realtime/api.js new file mode 100644 index 0000000..a99c54f --- /dev/null +++ b/server-deno/realtime/api.js @@ -0,0 +1,208 @@ +import { RealtimeEventHandler } from './event_handler.js'; +import { RealtimeUtils } from './utils.js'; + +export class RealtimeAPI extends RealtimeEventHandler { + /** + * Create a new RealtimeAPI instance + * @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings] + * @returns {RealtimeAPI} + */ + constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) { + super(); + this.defaultUrl = 'wss://api.openai.com/v1/realtime'; + this.url = url || this.defaultUrl; + this.apiKey = apiKey || null; + this.debug = !!debug; + this.ws = null; + if (globalThis.document && this.apiKey) { + if (!dangerouslyAllowAPIKeyInBrowser) { + throw new Error( + `Can not provide API key in the browser without "dangerouslyAllowAPIKeyInBrowser" set to true`, + ); + } + } + } + + /** + * Tells us whether or not the WebSocket is connected + * @returns {boolean} + */ + isConnected() { + return !!this.ws; + } + + /** + * Writes WebSocket logs to console + * @param {...any} args + * @returns {true} + */ + log(...args) { + const date = new Date().toISOString(); + const logs = [`[Websocket/${date}]`].concat(args).map((arg) => { + if (typeof arg === 'object' && arg !== null) { + return JSON.stringify(arg, null, 2); + } else { + return arg; + } + }); + if (this.debug) { + console.log(...logs); + } + return true; + } + + /** + * Connects to Realtime API Websocket Server + * @param {{model?: string}} [settings] + * @returns {Promise} + */ + async connect({ model } = { model: 'gpt-4o-realtime-preview-2024-10-01' }) { + if (!this.apiKey && this.url === this.defaultUrl) { + console.warn(`No apiKey provided for connection to "${this.url}"`); + } + if (this.isConnected()) { + throw new Error(`Already connected`); + } + if (globalThis.WebSocket) { + /** + * Web browser + */ + if (globalThis.document && this.apiKey) { + console.warn( + 'Warning: Connecting using API key in the browser, this is not recommended', + ); + } + const WebSocket = globalThis.WebSocket; + const ws = new WebSocket(`${this.url}${model ? `?model=${model}` : ''}`, [ + 'realtime', + `openai-insecure-api-key.${this.apiKey}`, + 'openai-beta.realtime-v1', + ]); + ws.addEventListener('message', (event) => { + const message = JSON.parse(event.data); + this.receive(message.type, message); + }); + return new Promise((resolve, reject) => { + const connectionErrorHandler = () => { + this.disconnect(ws); + reject(new Error(`Could not connect to "${this.url}"`)); + }; + ws.addEventListener('error', connectionErrorHandler); + ws.addEventListener('open', () => { + this.log(`Connected to "${this.url}"`); + ws.removeEventListener('error', connectionErrorHandler); + ws.addEventListener('error', () => { + this.disconnect(ws); + this.log(`Error, disconnected from "${this.url}"`); + this.dispatch('close', { error: true }); + }); + ws.addEventListener('close', () => { + this.disconnect(ws); + this.log(`Disconnected from "${this.url}"`); + this.dispatch('close', { error: false }); + }); + this.ws = ws; + resolve(true); + }); + }); + } else { + /** + * Node.js + */ + const moduleName = 'ws'; + const wsModule = await import(/* webpackIgnore: true */ moduleName); + const WebSocket = wsModule.default; + const ws = new WebSocket( + 'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01', + [], + { + finishRequest: (request) => { + // Auth + request.setHeader('Authorization', `Bearer ${this.apiKey}`); + request.setHeader('OpenAI-Beta', 'realtime=v1'); + request.end(); + }, + }, + ); + ws.on('message', (data) => { + const message = JSON.parse(data.toString()); + this.receive(message.type, message); + }); + return new Promise((resolve, reject) => { + const connectionErrorHandler = () => { + this.disconnect(ws); + reject(new Error(`Could not connect to "${this.url}"`)); + }; + ws.on('error', connectionErrorHandler); + ws.on('open', () => { + this.log(`Connected to "${this.url}"`); + ws.removeListener('error', connectionErrorHandler); + ws.on('error', () => { + this.disconnect(ws); + this.log(`Error, disconnected from "${this.url}"`); + this.dispatch('close', { error: true }); + }); + ws.on('close', () => { + this.disconnect(ws); + this.log(`Disconnected from "${this.url}"`); + this.dispatch('close', { error: false }); + }); + this.ws = ws; + resolve(true); + }); + }); + } + } + + /** + * Disconnects from Realtime API server + * @param {WebSocket} [ws] + * @returns {true} + */ + disconnect(ws) { + if (!ws || this.ws === ws) { + this.ws && this.ws.close(); + this.ws = null; + return true; + } + } + + /** + * Receives an event from WebSocket and dispatches as "server.{eventName}" and "server.*" events + * @param {string} eventName + * @param {{[key: string]: any}} event + * @returns {true} + */ + receive(eventName, event) { + this.log(`received:`, eventName, event); + this.dispatch(`server.${eventName}`, event); + this.dispatch('server.*', event); + return true; + } + + /** + * Sends an event to WebSocket and dispatches as "client.{eventName}" and "client.*" events + * @param {string} eventName + * @param {{[key: string]: any}} event + * @returns {true} + */ + send(eventName, data) { + if (!this.isConnected()) { + throw new Error(`RealtimeAPI is not connected`); + } + data = data || {}; + if (typeof data !== 'object') { + throw new Error(`data must be an object`); + } + const event = { + event_id: RealtimeUtils.generateId('evt_'), + type: eventName, + ...data, + }; + this.dispatch(`client.${eventName}`, event); + this.dispatch('client.*', event); + this.log(`sent:`, eventName, event); + this.ws.send(JSON.stringify(event)); + return true; + } +} diff --git a/server-deno/realtime/client.js b/server-deno/realtime/client.js new file mode 100644 index 0000000..23ae754 --- /dev/null +++ b/server-deno/realtime/client.js @@ -0,0 +1,676 @@ +import { RealtimeEventHandler } from './event_handler.js'; +import { RealtimeAPI } from './api.js'; +import { RealtimeConversation } from './conversation.js'; +import { RealtimeUtils } from './utils.js'; + +/** + * Valid audio formats + * @typedef {"pcm16"|"g711_ulaw"|"g711_alaw"} AudioFormatType + */ + +/** + * @typedef {Object} AudioTranscriptionType + * @property {"whisper-1"} model + */ + +/** + * @typedef {Object} TurnDetectionServerVadType + * @property {"server_vad"} type + * @property {number} [threshold] + * @property {number} [prefix_padding_ms] + * @property {number} [silence_duration_ms] + */ + +/** + * Tool definitions + * @typedef {Object} ToolDefinitionType + * @property {"function"} [type] + * @property {string} name + * @property {string} description + * @property {{[key: string]: any}} parameters + */ + +/** + * @typedef {Object} SessionResourceType + * @property {string} [model] + * @property {string[]} [modalities] + * @property {string} [instructions] + * @property {"alloy"|"ash"|"ballad"|"coral"|"echo"|"sage"|"shimmer"|"verse"} [voice] + * @property {AudioFormatType} [input_audio_format] + * @property {AudioFormatType} [output_audio_format] + * @property {AudioTranscriptionType|null} [input_audio_transcription] + * @property {TurnDetectionServerVadType|null} [turn_detection] + * @property {ToolDefinitionType[]} [tools] + * @property {"auto"|"none"|"required"|{type:"function",name:string}} [tool_choice] + * @property {number} [temperature] + * @property {number|"inf"} [max_response_output_tokens] + */ + +/** + * @typedef {"in_progress"|"completed"|"incomplete"} ItemStatusType + */ + +/** + * @typedef {Object} InputTextContentType + * @property {"input_text"} type + * @property {string} text + */ + +/** + * @typedef {Object} InputAudioContentType + * @property {"input_audio"} type + * @property {string} [audio] base64-encoded audio data + * @property {string|null} [transcript] + */ + +/** + * @typedef {Object} TextContentType + * @property {"text"} type + * @property {string} text + */ + +/** + * @typedef {Object} AudioContentType + * @property {"audio"} type + * @property {string} [audio] base64-encoded audio data + * @property {string|null} [transcript] + */ + +/** + * @typedef {Object} SystemItemType + * @property {string|null} [previous_item_id] + * @property {"message"} type + * @property {ItemStatusType} status + * @property {"system"} role + * @property {Array} content + */ + +/** + * @typedef {Object} UserItemType + * @property {string|null} [previous_item_id] + * @property {"message"} type + * @property {ItemStatusType} status + * @property {"user"} role + * @property {Array} content + */ + +/** + * @typedef {Object} AssistantItemType + * @property {string|null} [previous_item_id] + * @property {"message"} type + * @property {ItemStatusType} status + * @property {"assistant"} role + * @property {Array} content + */ + +/** + * @typedef {Object} FunctionCallItemType + * @property {string|null} [previous_item_id] + * @property {"function_call"} type + * @property {ItemStatusType} status + * @property {string} call_id + * @property {string} name + * @property {string} arguments + */ + +/** + * @typedef {Object} FunctionCallOutputItemType + * @property {string|null} [previous_item_id] + * @property {"function_call_output"} type + * @property {string} call_id + * @property {string} output + */ + +/** + * @typedef {Object} FormattedToolType + * @property {"function"} type + * @property {string} name + * @property {string} call_id + * @property {string} arguments + */ + +/** + * @typedef {Object} FormattedPropertyType + * @property {Int16Array} [audio] + * @property {string} [text] + * @property {string} [transcript] + * @property {FormattedToolType} [tool] + * @property {string} [output] + * @property {any} [file] + */ + +/** + * @typedef {Object} FormattedItemType + * @property {string} id + * @property {string} object + * @property {"user"|"assistant"|"system"} [role] + * @property {FormattedPropertyType} formatted + */ + +/** + * @typedef {SystemItemType|UserItemType|AssistantItemType|FunctionCallItemType|FunctionCallOutputItemType} BaseItemType + */ + +/** + * @typedef {FormattedItemType & BaseItemType} ItemType + */ + +/** + * @typedef {Object} IncompleteResponseStatusType + * @property {"incomplete"} type + * @property {"interruption"|"max_output_tokens"|"content_filter"} reason + */ + +/** + * @typedef {Object} FailedResponseStatusType + * @property {"failed"} type + * @property {{code: string, message: string}|null} error + */ + +/** + * @typedef {Object} UsageType + * @property {number} total_tokens + * @property {number} input_tokens + * @property {number} output_tokens + */ + +/** + * @typedef {Object} ResponseResourceType + * @property {"in_progress"|"completed"|"incomplete"|"cancelled"|"failed"} status + * @property {IncompleteResponseStatusType|FailedResponseStatusType|null} status_details + * @property {ItemType[]} output + * @property {UsageType|null} usage + */ + +/** + * RealtimeClient Class + * @class + */ +export class RealtimeClient extends RealtimeEventHandler { + /** + * Create a new RealtimeClient instance + * @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings] + */ + constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) { + super(); + this.defaultSessionConfig = { + modalities: ['text', 'audio'], + instructions: '', + voice: 'verse', + input_audio_format: 'pcm16', + output_audio_format: 'pcm16', + input_audio_transcription: null, + turn_detection: null, + tools: [], + tool_choice: 'auto', + temperature: 0.8, + max_response_output_tokens: 4096, + }; + this.sessionConfig = {}; + this.transcriptionModels = [ + { + model: 'whisper-1', + }, + ]; + this.defaultServerVadConfig = { + type: 'server_vad', + threshold: 0.5, // 0.0 to 1.0, + prefix_padding_ms: 300, // How much audio to include in the audio stream before the speech starts. + silence_duration_ms: 200, // How long to wait to mark the speech as stopped. + }; + this.realtime = new RealtimeAPI({ + url, + apiKey, + dangerouslyAllowAPIKeyInBrowser, + debug, + }); + this.conversation = new RealtimeConversation(); + this._resetConfig(); + this._addAPIEventHandlers(); + } + + /** + * Resets sessionConfig and conversationConfig to default + * @private + * @returns {true} + */ + _resetConfig() { + this.sessionCreated = false; + this.tools = {}; + this.sessionConfig = JSON.parse(JSON.stringify(this.defaultSessionConfig)); + this.inputAudioBuffer = new Int16Array(0); + return true; + } + + /** + * Sets up event handlers for a fully-functional application control flow + * @private + * @returns {true} + */ + _addAPIEventHandlers() { + // Event Logging handlers + this.realtime.on('client.*', (event) => { + const realtimeEvent = { + time: new Date().toISOString(), + source: 'client', + event: event, + }; + this.dispatch('realtime.event', realtimeEvent); + }); + this.realtime.on('server.*', (event) => { + const realtimeEvent = { + time: new Date().toISOString(), + source: 'server', + event: event, + }; + this.dispatch('realtime.event', realtimeEvent); + }); + + // Handles session created event, can optionally wait for it + this.realtime.on( + 'server.session.created', + () => (this.sessionCreated = true), + ); + + // Setup for application control flow + const handler = (event, ...args) => { + const { item, delta } = this.conversation.processEvent(event, ...args); + return { item, delta }; + }; + const handlerWithDispatch = (event, ...args) => { + const { item, delta } = handler(event, ...args); + if (item) { + // FIXME: If statement is only here because item.input_audio_transcription.completed + // can fire before `item.created`, resulting in empty item. + // This happens in VAD mode with empty audio + this.dispatch('conversation.updated', { item, delta }); + } + return { item, delta }; + }; + const callTool = async (tool) => { + try { + const jsonArguments = JSON.parse(tool.arguments); + const toolConfig = this.tools[tool.name]; + if (!toolConfig) { + throw new Error(`Tool "${tool.name}" has not been added`); + } + const result = await toolConfig.handler(jsonArguments); + this.realtime.send('conversation.item.create', { + item: { + type: 'function_call_output', + call_id: tool.call_id, + output: JSON.stringify(result), + }, + }); + } catch (e) { + this.realtime.send('conversation.item.create', { + item: { + type: 'function_call_output', + call_id: tool.call_id, + output: JSON.stringify({ error: e.message }), + }, + }); + } + this.createResponse(); + }; + + // Handlers to update internal conversation state + this.realtime.on('server.response.created', handler); + this.realtime.on('server.response.output_item.added', handler); + this.realtime.on('server.response.content_part.added', handler); + this.realtime.on('server.input_audio_buffer.speech_started', (event) => { + handler(event); + this.dispatch('conversation.interrupted'); + }); + this.realtime.on( + 'server.input_audio_buffer.speech_stopped', + (event) => handler(event, this.inputAudioBuffer), + ); + + // Handlers to update application state + this.realtime.on('server.conversation.item.created', (event) => { + const { item } = handlerWithDispatch(event); + this.dispatch('conversation.item.appended', { item }); + if (item.status === 'completed') { + this.dispatch('conversation.item.completed', { item }); + } + }); + this.realtime.on('server.conversation.item.truncated', handlerWithDispatch); + this.realtime.on('server.conversation.item.deleted', handlerWithDispatch); + this.realtime.on( + 'server.conversation.item.input_audio_transcription.completed', + handlerWithDispatch, + ); + this.realtime.on( + 'server.response.audio_transcript.delta', + handlerWithDispatch, + ); + this.realtime.on('server.response.audio.delta', handlerWithDispatch); + this.realtime.on('server.response.text.delta', handlerWithDispatch); + this.realtime.on( + 'server.response.function_call_arguments.delta', + handlerWithDispatch, + ); + this.realtime.on('server.response.output_item.done', async (event) => { + const { item } = handlerWithDispatch(event); + if (item.status === 'completed') { + this.dispatch('conversation.item.completed', { item }); + } + if (item.formatted.tool) { + callTool(item.formatted.tool); + } + }); + + return true; + } + + /** + * Tells us whether the realtime socket is connected and the session has started + * @returns {boolean} + */ + isConnected() { + return this.realtime.isConnected(); + } + + /** + * Resets the client instance entirely: disconnects and clears active config + * @returns {true} + */ + reset() { + this.disconnect(); + this.clearEventHandlers(); + this.realtime.clearEventHandlers(); + this._resetConfig(); + this._addAPIEventHandlers(); + return true; + } + + /** + * Connects to the Realtime WebSocket API + * Updates session config and conversation config + * @returns {Promise} + */ + async connect({ + model = 'gpt-4o-mini-realtime-preview-2024-12-17', + turn_detection = null, + voice, + instructions, + input_audio_transcription, + }) { + if (this.isConnected()) { + throw new Error(`Already connected, use .disconnect() first`); + } + await this.realtime.connect({ + model, + }); + this.updateSession({ + voice, + turn_detection, + instructions, + input_audio_transcription, + }); + return true; + } + + /** + * Waits for a session.created event to be executed before proceeding + * @returns {Promise} + */ + async waitForSessionCreated() { + if (!this.isConnected()) { + throw new Error(`Not connected, use .connect() first`); + } + while (!this.sessionCreated) { + await new Promise((r) => setTimeout(() => r(), 1)); + } + return true; + } + + /** + * Disconnects from the Realtime API and clears the conversation history + */ + disconnect() { + console.log('foobar disconnecting from realtime client'); + this.sessionCreated = false; + this.realtime.isConnected() && this.realtime.disconnect(); + this.conversation.clear(); + } + + /** + * Gets the active turn detection mode + * @returns {"server_vad"|null} + */ + getTurnDetectionType() { + return this.sessionConfig.turn_detection?.type || null; + } + + /** + * Add a tool and handler + * @param {ToolDefinitionType} definition + * @param {function} handler + * @returns {{definition: ToolDefinitionType, handler: function}} + */ + addTool(definition, handler) { + if (!definition?.name) { + throw new Error(`Missing tool name in definition`); + } + const name = definition?.name; + if (this.tools[name]) { + throw new Error( + `Tool "${name}" already added. Please use .removeTool("${name}") before trying to add again.`, + ); + } + if (typeof handler !== 'function') { + throw new Error(`Tool "${name}" handler must be a function`); + } + this.tools[name] = { definition, handler }; + this.updateSession(); + return this.tools[name]; + } + + /** + * Removes a tool + * @param {string} name + * @returns {true} + */ + removeTool(name) { + if (!this.tools[name]) { + throw new Error(`Tool "${name}" does not exist, can not be removed.`); + } + delete this.tools[name]; + return true; + } + + /** + * Deletes an item + * @param {string} id + * @returns {true} + */ + deleteItem(id) { + this.realtime.send('conversation.item.delete', { item_id: id }); + return true; + } + + /** + * Updates session configuration + * If the client is not yet connected, will save details and instantiate upon connection + * @param {SessionResourceType} [sessionConfig] + */ + updateSession({ + modalities = void 0, + instructions = void 0, + voice = void 0, + input_audio_format = void 0, + output_audio_format = void 0, + input_audio_transcription = void 0, + turn_detection = void 0, + tools = void 0, + tool_choice = void 0, + temperature = void 0, + max_response_output_tokens = void 0, + } = {}) { + modalities !== void 0 && (this.sessionConfig.modalities = modalities); + instructions !== void 0 && (this.sessionConfig.instructions = instructions); + voice !== void 0 && (this.sessionConfig.voice = voice); + input_audio_format !== void 0 && + (this.sessionConfig.input_audio_format = input_audio_format); + output_audio_format !== void 0 && + (this.sessionConfig.output_audio_format = output_audio_format); + input_audio_transcription !== void 0 && + (this.sessionConfig.input_audio_transcription = input_audio_transcription); + turn_detection !== void 0 && + (this.sessionConfig.turn_detection = turn_detection); + tools !== void 0 && (this.sessionConfig.tools = tools); + tool_choice !== void 0 && (this.sessionConfig.tool_choice = tool_choice); + temperature !== void 0 && (this.sessionConfig.temperature = temperature); + max_response_output_tokens !== void 0 && + (this.sessionConfig.max_response_output_tokens = max_response_output_tokens); + // Load tools from tool definitions + already loaded tools + const useTools = [].concat( + (tools || []).map((toolDefinition) => { + const definition = { + type: 'function', + ...toolDefinition, + }; + if (this.tools[definition?.name]) { + throw new Error( + `Tool "${definition?.name}" has already been defined`, + ); + } + return definition; + }), + Object.keys(this.tools).map((key) => { + return { + type: 'function', + ...this.tools[key].definition, + }; + }), + ); + const session = { ...this.sessionConfig }; + session.tools = useTools; + if (this.realtime.isConnected()) { + this.realtime.send('session.update', { session }); + } + return true; + } + + /** + * Sends user message content and generates a response + * @param {Array} content + * @returns {true} + */ + sendUserMessageContent(content = []) { + if (content.length) { + for (const c of content) { + if (c.type === 'input_audio') { + if (c.audio instanceof ArrayBuffer || c.audio instanceof Int16Array) { + c.audio = RealtimeUtils.arrayBufferToBase64(c.audio); + } + } + } + this.realtime.send('conversation.item.create', { + item: { + type: 'message', + role: 'user', + content, + }, + }); + } + this.createResponse(); + return true; + } + + /** + * Appends user audio to the existing audio buffer + * @param {Int16Array|ArrayBuffer} arrayBuffer + * @returns {true} + */ + appendInputAudio(arrayBuffer) { + if (arrayBuffer.byteLength > 0) { + this.realtime.send('input_audio_buffer.append', { + audio: RealtimeUtils.arrayBufferToBase64(arrayBuffer), + }); + this.inputAudioBuffer = RealtimeUtils.mergeInt16Arrays( + this.inputAudioBuffer, + arrayBuffer, + ); + } + return true; + } + + /** + * Forces a model response generation + * @returns {true} + */ + createResponse() { + if ( + this.getTurnDetectionType() === null && + this.inputAudioBuffer.byteLength > 0 + ) { + this.realtime.send('input_audio_buffer.commit'); + this.conversation.queueInputAudio(this.inputAudioBuffer); + this.inputAudioBuffer = new Int16Array(0); + } + this.realtime.send('response.create'); + return true; + } + + /** + * Cancels the ongoing server generation and truncates ongoing generation, if applicable + * If no id provided, will simply call `cancel_generation` command + * @param {string} id The id of the message to cancel + * @param {number} [sampleCount] The number of samples to truncate past for the ongoing generation + * @returns {{item: (AssistantItemType | null)}} + */ + cancelResponse(id, sampleCount = 0) { + if (!id) { + this.realtime.send('response.cancel'); + return { item: null }; + } else if (id) { + const item = this.conversation.getItem(id); + if (!item) { + throw new Error(`Could not find item "${id}"`); + } + if (item.type !== 'message') { + throw new Error(`Can only cancelResponse messages with type "message"`); + } else if (item.role !== 'assistant') { + throw new Error( + `Can only cancelResponse messages with role "assistant"`, + ); + } + this.realtime.send('response.cancel'); + const audioIndex = item.content.findIndex((c) => c.type === 'audio'); + if (audioIndex === -1) { + throw new Error(`Could not find audio on item to cancel`); + } + this.realtime.send('conversation.item.truncate', { + item_id: id, + content_index: audioIndex, + audio_end_ms: Math.floor( + (sampleCount / this.conversation.defaultFrequency) * 1000, + ), + }); + return { item }; + } + } + + /** + * Utility for waiting for the next `conversation.item.appended` event to be triggered by the server + * @returns {Promise<{item: ItemType}>} + */ + async waitForNextItem() { + const event = await this.waitForNext('conversation.item.appended'); + const { item } = event; + return { item }; + } + + /** + * Utility for waiting for the next `conversation.item.completed` event to be triggered by the server + * @returns {Promise<{item: ItemType}>} + */ + async waitForNextCompletedItem() { + const event = await this.waitForNext('conversation.item.completed'); + const { item } = event; + return { item }; + } +} diff --git a/server-deno/realtime/conversation.js b/server-deno/realtime/conversation.js new file mode 100644 index 0000000..9250221 --- /dev/null +++ b/server-deno/realtime/conversation.js @@ -0,0 +1,314 @@ +import { RealtimeUtils } from './utils.js'; + +/** + * Contains text and audio information about a item + * Can also be used as a delta + * @typedef {Object} ItemContentDeltaType + * @property {string} [text] + * @property {Int16Array} [audio] + * @property {string} [arguments] + * @property {string} [transcript] + */ + +/** + * RealtimeConversation holds conversation history + * and performs event validation for RealtimeAPI + * @class + */ +export class RealtimeConversation { + defaultFrequency = 24_000; // 24,000 Hz + + EventProcessors = { + 'conversation.item.created': (event) => { + const { item } = event; + // deep copy values + const newItem = JSON.parse(JSON.stringify(item)); + if (!this.itemLookup[newItem.id]) { + this.itemLookup[newItem.id] = newItem; + this.items.push(newItem); + } + newItem.formatted = {}; + newItem.formatted.audio = new Int16Array(0); + newItem.formatted.text = ''; + newItem.formatted.transcript = ''; + // If we have a speech item, can populate audio + if (this.queuedSpeechItems[newItem.id]) { + newItem.formatted.audio = this.queuedSpeechItems[newItem.id].audio; + delete this.queuedSpeechItems[newItem.id]; // free up some memory + } + // Populate formatted text if it comes out on creation + if (newItem.content) { + const textContent = newItem.content.filter((c) => + ['text', 'input_text'].includes(c.type) + ); + for (const content of textContent) { + newItem.formatted.text += content.text; + } + } + // If we have a transcript item, can pre-populate transcript + if (this.queuedTranscriptItems[newItem.id]) { + newItem.formatted.transcript = this.queuedTranscriptItems.transcript; + delete this.queuedTranscriptItems[newItem.id]; + } + if (newItem.type === 'message') { + if (newItem.role === 'user') { + newItem.status = 'completed'; + if (this.queuedInputAudio) { + newItem.formatted.audio = this.queuedInputAudio; + this.queuedInputAudio = null; + } + } else { + newItem.status = 'in_progress'; + } + } else if (newItem.type === 'function_call') { + newItem.formatted.tool = { + type: 'function', + name: newItem.name, + call_id: newItem.call_id, + arguments: '', + }; + newItem.status = 'in_progress'; + } else if (newItem.type === 'function_call_output') { + newItem.status = 'completed'; + newItem.formatted.output = newItem.output; + } + return { item: newItem, delta: null }; + }, + 'conversation.item.truncated': (event) => { + const { item_id, audio_end_ms } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error(`item.truncated: Item "${item_id}" not found`); + } + const endIndex = Math.floor( + (audio_end_ms * this.defaultFrequency) / 1000, + ); + item.formatted.transcript = ''; + item.formatted.audio = item.formatted.audio.slice(0, endIndex); + return { item, delta: null }; + }, + 'conversation.item.deleted': (event) => { + const { item_id } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error(`item.deleted: Item "${item_id}" not found`); + } + delete this.itemLookup[item.id]; + const index = this.items.indexOf(item); + if (index > -1) { + this.items.splice(index, 1); + } + return { item, delta: null }; + }, + 'conversation.item.input_audio_transcription.completed': (event) => { + const { item_id, content_index, transcript } = event; + const item = this.itemLookup[item_id]; + // We use a single space to represent an empty transcript for .formatted values + // Otherwise it looks like no transcript provided + const formattedTranscript = transcript || ' '; + if (!item) { + // We can receive transcripts in VAD mode before item.created + // This happens specifically when audio is empty + this.queuedTranscriptItems[item_id] = { + transcript: formattedTranscript, + }; + return { item: null, delta: null }; + } else { + item.content[content_index].transcript = transcript; + item.formatted.transcript = formattedTranscript; + return { item, delta: { transcript } }; + } + }, + 'input_audio_buffer.speech_started': (event) => { + const { item_id, audio_start_ms } = event; + this.queuedSpeechItems[item_id] = { audio_start_ms }; + return { item: null, delta: null }; + }, + 'input_audio_buffer.speech_stopped': (event, inputAudioBuffer) => { + const { item_id, audio_end_ms } = event; + if (!this.queuedSpeechItems[item_id]) { + this.queuedSpeechItems[item_id] = { audio_start_ms: audio_end_ms }; + } + const speech = this.queuedSpeechItems[item_id]; + speech.audio_end_ms = audio_end_ms; + if (inputAudioBuffer) { + const startIndex = Math.floor( + (speech.audio_start_ms * this.defaultFrequency) / 1000, + ); + const endIndex = Math.floor( + (speech.audio_end_ms * this.defaultFrequency) / 1000, + ); + speech.audio = inputAudioBuffer.slice(startIndex, endIndex); + } + return { item: null, delta: null }; + }, + 'response.created': (event) => { + const { response } = event; + if (!this.responseLookup[response.id]) { + this.responseLookup[response.id] = response; + this.responses.push(response); + } + return { item: null, delta: null }; + }, + 'response.output_item.added': (event) => { + const { response_id, item } = event; + const response = this.responseLookup[response_id]; + if (!response) { + throw new Error( + `response.output_item.added: Response "${response_id}" not found`, + ); + } + response.output.push(item.id); + return { item: null, delta: null }; + }, + 'response.output_item.done': (event) => { + const { item } = event; + if (!item) { + throw new Error(`response.output_item.done: Missing "item"`); + } + const foundItem = this.itemLookup[item.id]; + if (!foundItem) { + throw new Error( + `response.output_item.done: Item "${item.id}" not found`, + ); + } + foundItem.status = item.status; + return { item: foundItem, delta: null }; + }, + 'response.content_part.added': (event) => { + const { item_id, part } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error( + `response.content_part.added: Item "${item_id}" not found`, + ); + } + item.content.push(part); + return { item, delta: null }; + }, + 'response.audio_transcript.delta': (event) => { + const { item_id, content_index, delta } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error( + `response.audio_transcript.delta: Item "${item_id}" not found`, + ); + } + item.content[content_index].transcript += delta; + item.formatted.transcript += delta; + return { item, delta: { transcript: delta } }; + }, + 'response.audio.delta': (event) => { + const { item_id, content_index, delta } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error(`response.audio.delta: Item "${item_id}" not found`); + } + // This never gets renderered, we care about the file data instead + // item.content[content_index].audio += delta; + const arrayBuffer = RealtimeUtils.base64ToArrayBuffer(delta); + const appendValues = new Int16Array(arrayBuffer); + item.formatted.audio = RealtimeUtils.mergeInt16Arrays( + item.formatted.audio, + appendValues, + ); + return { item, delta: { audio: appendValues } }; + }, + 'response.text.delta': (event) => { + const { item_id, content_index, delta } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error(`response.text.delta: Item "${item_id}" not found`); + } + item.content[content_index].text += delta; + item.formatted.text += delta; + return { item, delta: { text: delta } }; + }, + 'response.function_call_arguments.delta': (event) => { + const { item_id, delta } = event; + const item = this.itemLookup[item_id]; + if (!item) { + throw new Error( + `response.function_call_arguments.delta: Item "${item_id}" not found`, + ); + } + item.arguments += delta; + item.formatted.tool.arguments += delta; + return { item, delta: { arguments: delta } }; + }, + }; + + /** + * Create a new RealtimeConversation instance + * @returns {RealtimeConversation} + */ + constructor() { + this.clear(); + } + + /** + * Clears the conversation history and resets to default + * @returns {true} + */ + clear() { + this.itemLookup = {}; + this.items = []; + this.responseLookup = {}; + this.responses = []; + this.queuedSpeechItems = {}; + this.queuedTranscriptItems = {}; + this.queuedInputAudio = null; + return true; + } + + /** + * Queue input audio for manual speech event + * @param {Int16Array} inputAudio + * @returns {Int16Array} + */ + queueInputAudio(inputAudio) { + this.queuedInputAudio = inputAudio; + return inputAudio; + } + + /** + * Process an event from the WebSocket server and compose items + * @param {Object} event + * @param {...any} args + * @returns {item: import('./client.js').ItemType | null, delta: ItemContentDeltaType | null} + */ + processEvent(event, ...args) { + if (!event.event_id) { + console.error(event); + throw new Error(`Missing "event_id" on event`); + } + if (!event.type) { + console.error(event); + throw new Error(`Missing "type" on event`); + } + const eventProcessor = this.EventProcessors[event.type]; + if (!eventProcessor) { + throw new Error( + `Missing conversation event processor for "${event.type}"`, + ); + } + return eventProcessor.call(this, event, ...args); + } + + /** + * Retrieves a item by id + * @param {string} id + * @returns {import('./client.js').ItemType} + */ + getItem(id) { + return this.itemLookup[id] || null; + } + + /** + * Retrieves all items in the conversation + * @returns {import('./client.js').ItemType[]} + */ + getItems() { + return this.items.slice(); + } +} diff --git a/server-deno/realtime/event_handler.js b/server-deno/realtime/event_handler.js new file mode 100644 index 0000000..864c8cb --- /dev/null +++ b/server-deno/realtime/event_handler.js @@ -0,0 +1,143 @@ +/** + * EventHandler callback + * @typedef {(event: {[key: string]: any}): void} EventHandlerCallbackType + */ + +const sleep = (t) => new Promise((r) => setTimeout(() => r(), t)); + +/** + * Inherited class for RealtimeAPI and RealtimeClient + * Adds basic event handling + * @class + */ +export class RealtimeEventHandler { + /** + * Create a new RealtimeEventHandler instance + * @returns {RealtimeEventHandler} + */ + constructor() { + this.eventHandlers = {}; + this.nextEventHandlers = {}; + } + + /** + * Clears all event handlers + * @returns {true} + */ + clearEventHandlers() { + this.eventHandlers = {}; + this.nextEventHandlers = {}; + return true; + } + + /** + * Listen to specific events + * @param {string} eventName The name of the event to listen to + * @param {EventHandlerCallbackType} callback Code to execute on event + * @returns {EventHandlerCallbackType} + */ + on(eventName, callback) { + this.eventHandlers[eventName] = this.eventHandlers[eventName] || []; + this.eventHandlers[eventName].push(callback); + return callback; + } + + /** + * Listen for the next event of a specified type + * @param {string} eventName The name of the event to listen to + * @param {EventHandlerCallbackType} callback Code to execute on event + * @returns {EventHandlerCallbackType} + */ + onNext(eventName, callback) { + this.nextEventHandlers[eventName] = this.nextEventHandlers[eventName] || []; + this.nextEventHandlers[eventName].push(callback); + return callback; + } + + /** + * Turns off event listening for specific events + * Calling without a callback will remove all listeners for the event + * @param {string} eventName + * @param {EventHandlerCallbackType} [callback] + * @returns {true} + */ + off(eventName, callback) { + const handlers = this.eventHandlers[eventName] || []; + if (callback) { + const index = handlers.indexOf(callback); + if (index === -1) { + throw new Error( + `Could not turn off specified event listener for "${eventName}": not found as a listener`, + ); + } + handlers.splice(index, 1); + } else { + delete this.eventHandlers[eventName]; + } + return true; + } + + /** + * Turns off event listening for the next event of a specific type + * Calling without a callback will remove all listeners for the next event + * @param {string} eventName + * @param {EventHandlerCallbackType} [callback] + * @returns {true} + */ + offNext(eventName, callback) { + const nextHandlers = this.nextEventHandlers[eventName] || []; + if (callback) { + const index = nextHandlers.indexOf(callback); + if (index === -1) { + throw new Error( + `Could not turn off specified next event listener for "${eventName}": not found as a listener`, + ); + } + nextHandlers.splice(index, 1); + } else { + delete this.nextEventHandlers[eventName]; + } + return true; + } + + /** + * Waits for next event of a specific type and returns the payload + * @param {string} eventName + * @param {number|null} [timeout] + * @returns {Promise<{[key: string]: any}|null>} + */ + async waitForNext(eventName, timeout = null) { + const t0 = Date.now(); + let nextEvent; + this.onNext(eventName, (event) => (nextEvent = event)); + while (!nextEvent) { + if (timeout) { + const t1 = Date.now(); + if (t1 - t0 > timeout) { + return null; + } + } + await sleep(1); + } + return nextEvent; + } + + /** + * Executes all events in the order they were added, with .on() event handlers executing before .onNext() handlers + * @param {string} eventName + * @param {any} event + * @returns {true} + */ + dispatch(eventName, event) { + const handlers = [].concat(this.eventHandlers[eventName] || []); + for (const handler of handlers) { + handler(event); + } + const nextHandlers = [].concat(this.nextEventHandlers[eventName] || []); + for (const nextHandler of nextHandlers) { + nextHandler(event); + } + delete this.nextEventHandlers[eventName]; + return true; + } +} diff --git a/server-deno/realtime/utils.js b/server-deno/realtime/utils.js new file mode 100644 index 0000000..8e6e1dd --- /dev/null +++ b/server-deno/realtime/utils.js @@ -0,0 +1,102 @@ +const atob = globalThis.atob; +const btoa = globalThis.btoa; + +/** + * Basic utilities for the RealtimeAPI + * @class + */ +export class RealtimeUtils { + /** + * Converts Float32Array of amplitude data to ArrayBuffer in Int16Array format + * @param {Float32Array} float32Array + * @returns {ArrayBuffer} + */ + static floatTo16BitPCM(float32Array) { + const buffer = new ArrayBuffer(float32Array.length * 2); + const view = new DataView(buffer); + let offset = 0; + for (let i = 0; i < float32Array.length; i++, offset += 2) { + let s = Math.max(-1, Math.min(1, float32Array[i])); + view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true); + } + return buffer; + } + + /** + * Converts a base64 string to an ArrayBuffer + * @param {string} base64 + * @returns {ArrayBuffer} + */ + static base64ToArrayBuffer(base64) { + const binaryString = atob(base64); + const len = binaryString.length; + const bytes = new Uint8Array(len); + for (let i = 0; i < len; i++) { + bytes[i] = binaryString.charCodeAt(i); + } + return bytes.buffer; + } + + /** + * Converts an ArrayBuffer, Int16Array or Float32Array to a base64 string + * @param {ArrayBuffer|Int16Array|Float32Array} arrayBuffer + * @returns {string} + */ + static arrayBufferToBase64(arrayBuffer) { + if (arrayBuffer instanceof Float32Array) { + arrayBuffer = this.floatTo16BitPCM(arrayBuffer); + } else if (arrayBuffer instanceof Int16Array) { + arrayBuffer = arrayBuffer.buffer; + } + let binary = ''; + let bytes = new Uint8Array(arrayBuffer); + const chunkSize = 0x8000; // 32KB chunk size + for (let i = 0; i < bytes.length; i += chunkSize) { + let chunk = bytes.subarray(i, i + chunkSize); + binary += String.fromCharCode.apply(null, chunk); + } + return btoa(binary); + } + + /** + * Merge two Int16Arrays from Int16Arrays or ArrayBuffers + * @param {ArrayBuffer|Int16Array} left + * @param {ArrayBuffer|Int16Array} right + * @returns {Int16Array} + */ + static mergeInt16Arrays(left, right) { + if (left instanceof ArrayBuffer) { + left = new Int16Array(left); + } + if (right instanceof ArrayBuffer) { + right = new Int16Array(right); + } + if (!(left instanceof Int16Array) || !(right instanceof Int16Array)) { + throw new Error(`Both items must be Int16Array`); + } + const newValues = new Int16Array(left.length + right.length); + for (let i = 0; i < left.length; i++) { + newValues[i] = left[i]; + } + for (let j = 0; j < right.length; j++) { + newValues[left.length + j] = right[j]; + } + return newValues; + } + + /** + * Generates an id to send with events and messages + * @param {string} prefix + * @param {number} [length] + * @returns {string} + */ + static generateId(prefix, length = 21) { + // base58; non-repeating chars + const chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz'; + const str = Array(length - prefix.length) + .fill(0) + .map((_) => chars[Math.floor(Math.random() * chars.length)]) + .join(''); + return `${prefix}${str}`; + } +}