only register secure when in prod or elato mode
This commit is contained in:
akdeb
parent
14531d5f7a
commit
1697d1ebd0
10 changed files with 1461 additions and 35 deletions
|
|
@ -57,10 +57,10 @@ git clone git@github.com:akdeb/ElatoAI.git
|
|||
|
||||
2. **Start Supabase**
|
||||
|
||||
Install [Supabase CLI](https://supabase.com/docs/guides/local-development/cli/getting-started) and set up your Local Supabase Backend. From the root directory, run:
|
||||
Install [Supabase CLI](https://supabase.com/docs/guides/local-development/cli/getting-started) and set up your Local Supabase Backend. Make sure you have Docker Desktop set up. Then from the root directory, run:
|
||||
```bash
|
||||
brew install supabase/tap/supabase
|
||||
supabase start # Starts your local Supabase server with the default migrations and seed data.
|
||||
supabase start # This starts your local Supabase server with the default migrations and seed data.
|
||||
```
|
||||
|
||||
3. **Set up your NextJS Frontend**
|
||||
|
|
|
|||
|
|
@ -21,9 +21,6 @@ const char *server_certificate = R"EOF(
|
|||
|
||||
void markOTAUpdateComplete() {
|
||||
HTTPClient http;
|
||||
WiFiClientSecure client;
|
||||
client.setCACert(Vercel_CA_cert); // Using the existing server certificate
|
||||
|
||||
// Construct the JSON payload
|
||||
JsonDocument doc;
|
||||
doc["authToken"] = authTokenGlobal;
|
||||
|
|
@ -33,9 +30,11 @@ void markOTAUpdateComplete() {
|
|||
|
||||
// Initialize HTTPS connection with client
|
||||
#ifdef DEV_MODE
|
||||
http.begin("http://" + String(backend_server) + ":" + String(backend_port) + "/api/ota_update_handler");
|
||||
http.begin("http://" + String(backend_server) + ":" + String(backend_port) + "/api/ota_update_handler");
|
||||
#else
|
||||
http.begin(client, "https://" + String(backend_server) + "/api/ota_update_handler");
|
||||
WiFiClientSecure client;
|
||||
client.setCACert(Vercel_CA_cert); // Using the existing server certificate
|
||||
http.begin(client, "https://" + String(backend_server) + "/api/ota_update_handler");
|
||||
#endif
|
||||
|
||||
http.addHeader("Content-Type", "application/json");
|
||||
|
|
|
|||
|
|
@ -19,15 +19,15 @@ bool isDeviceRegistered() {
|
|||
}
|
||||
|
||||
HTTPClient http;
|
||||
WiFiClientSecure client;
|
||||
client.setCACert(Vercel_CA_cert);
|
||||
|
||||
#ifdef DEV_MODE
|
||||
http.begin("http://" + String(backend_server) + ":" + String(backend_port) +
|
||||
"/api/generate_auth_token?macAddress=" + WiFi.macAddress());
|
||||
http.begin("http://" + String(backend_server) + ":" + String(backend_port) +
|
||||
"/api/generate_auth_token?macAddress=" + WiFi.macAddress());
|
||||
#else
|
||||
http.begin(client, "https://" + String(backend_server) +
|
||||
"/api/generate_auth_token?macAddress=" + WiFi.macAddress());
|
||||
WiFiClientSecure client;
|
||||
client.setCACert(Vercel_CA_cert);
|
||||
http.begin(client, "https://" + String(backend_server) +
|
||||
"/api/generate_auth_token?macAddress=" + WiFi.macAddress());
|
||||
#endif
|
||||
|
||||
http.setTimeout(10000);
|
||||
|
|
|
|||
|
|
@ -1,8 +1,6 @@
|
|||
{
|
||||
"version": "4",
|
||||
"version": "5",
|
||||
"specifiers": {
|
||||
"jsr:@std/assert@1": "1.0.11",
|
||||
"jsr:@std/internal@^1.0.5": "1.0.5",
|
||||
"jsr:@supabase/supabase-js@2": "2.48.1",
|
||||
"jsr:@supabase/supabase-js@^2.48.1": "2.48.1",
|
||||
"npm:@evan/opus@^1.0.3": "1.0.3",
|
||||
|
|
@ -15,15 +13,6 @@
|
|||
"npm:ws@*": "8.18.0"
|
||||
},
|
||||
"jsr": {
|
||||
"@std/assert@1.0.11": {
|
||||
"integrity": "2461ef3c368fe88bc60e186e7744a93112f16fd110022e113a0849e94d1c83c1",
|
||||
"dependencies": [
|
||||
"jsr:@std/internal"
|
||||
]
|
||||
},
|
||||
"@std/internal@1.0.5": {
|
||||
"integrity": "54a546004f769c1ac9e025abd15a76b6671ddc9687e2313b67376125650dc7ba"
|
||||
},
|
||||
"@supabase/supabase-js@2.48.1": {
|
||||
"integrity": "747e85c2a546efcc9f84b343b08e10e3818d468ae214506b59dc65a36fd2d702",
|
||||
"dependencies": [
|
||||
|
|
@ -79,8 +68,8 @@
|
|||
"@supabase/node-fetch"
|
||||
]
|
||||
},
|
||||
"@types/node@22.12.0": {
|
||||
"integrity": "sha512-Fll2FZ1riMjNmlmJOdAyY5pUbkftXslB5DgEzlIuNaiWhXd00FhWxVC/r4yV/4wBb9JfImTu+jiSvXTkJ7F/gA==",
|
||||
"@types/node@22.13.0": {
|
||||
"integrity": "sha512-ClIbNe36lawluuvq3+YYhnIN2CELi+6q8NpnM7PYp4hBn/TatfboPgVSm2rwKRfnV2M+Ty9GWDFI64KEe+kysA==",
|
||||
"dependencies": [
|
||||
"undici-types"
|
||||
]
|
||||
|
|
@ -194,12 +183,7 @@
|
|||
"https://deno.land/x/jose@v5.9.6/util/decode_jwt.ts": "a6347360591bd60b076d115fc231e3ecd84aef319922a6c6e94554e68863e3af",
|
||||
"https://deno.land/x/jose@v5.9.6/util/decode_protected_header.ts": "3412c8aacffee93148478f14a0ebb97b2ff060944f49dd233223d262f592ee36",
|
||||
"https://deno.land/x/jose@v5.9.6/util/errors.ts": "b7ecf22990ce73a42650fd09666fb7174fcd91eec387aa26b361269c8022220f",
|
||||
"https://deno.land/x/jose@v5.9.6/util/runtime.ts": "088cae7df0d285065f141d880c5c21a572234997082337d1125d7f1603059642",
|
||||
"https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/api.js": "fb958c31e4c4e04dd4bee56c35f8cd0b69b72dd6abf297e3bf71a498f1f07f5a",
|
||||
"https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/client.js": "390bcc3d817f4c3deadcec4e65ce8e915e4f8ae19aee32d62de2e71f9adab65d",
|
||||
"https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/conversation.js": "f2ffef21963a669d705cfccdbc4ee92524e81f8680d0cb12914f18eebc2b1369",
|
||||
"https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/event_handler.js": "0eeea0fd7d288b25995233a644632b61c84d5af196ba5d0422d0b4040058c595",
|
||||
"https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/utils.js": "aecdf22b2530c1b5763a5888d43f3bdc76266f63ed0628c35ab54518e7dd436b"
|
||||
"https://deno.land/x/jose@v5.9.6/util/runtime.ts": "088cae7df0d285065f141d880c5c21a572234997082337d1125d7f1603059642"
|
||||
},
|
||||
"workspace": {
|
||||
"dependencies": [
|
||||
|
|
|
|||
|
|
@ -7,8 +7,8 @@ import type {
|
|||
WebSocketServer as _WebSocketServer,
|
||||
} from "npm:@types/ws";
|
||||
|
||||
import { RealtimeClient } from "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/client.js";
|
||||
import { RealtimeUtils } from "https://raw.githubusercontent.com/akdeb/openai-realtime-api-beta/refs/heads/main/lib/utils.js";
|
||||
import { RealtimeClient } from "./realtime/client.js";
|
||||
import { RealtimeUtils } from "./realtime/utils.js";
|
||||
import { authenticateUser } from "./utils.ts";
|
||||
import {
|
||||
addConversation,
|
||||
|
|
|
|||
208
server-deno/realtime/api.js
Normal file
208
server-deno/realtime/api.js
Normal file
|
|
@ -0,0 +1,208 @@
|
|||
import { RealtimeEventHandler } from './event_handler.js';
|
||||
import { RealtimeUtils } from './utils.js';
|
||||
|
||||
export class RealtimeAPI extends RealtimeEventHandler {
|
||||
/**
|
||||
* Create a new RealtimeAPI instance
|
||||
* @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
|
||||
* @returns {RealtimeAPI}
|
||||
*/
|
||||
constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
|
||||
super();
|
||||
this.defaultUrl = 'wss://api.openai.com/v1/realtime';
|
||||
this.url = url || this.defaultUrl;
|
||||
this.apiKey = apiKey || null;
|
||||
this.debug = !!debug;
|
||||
this.ws = null;
|
||||
if (globalThis.document && this.apiKey) {
|
||||
if (!dangerouslyAllowAPIKeyInBrowser) {
|
||||
throw new Error(
|
||||
`Can not provide API key in the browser without "dangerouslyAllowAPIKeyInBrowser" set to true`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells us whether or not the WebSocket is connected
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isConnected() {
|
||||
return !!this.ws;
|
||||
}
|
||||
|
||||
/**
|
||||
* Writes WebSocket logs to console
|
||||
* @param {...any} args
|
||||
* @returns {true}
|
||||
*/
|
||||
log(...args) {
|
||||
const date = new Date().toISOString();
|
||||
const logs = [`[Websocket/${date}]`].concat(args).map((arg) => {
|
||||
if (typeof arg === 'object' && arg !== null) {
|
||||
return JSON.stringify(arg, null, 2);
|
||||
} else {
|
||||
return arg;
|
||||
}
|
||||
});
|
||||
if (this.debug) {
|
||||
console.log(...logs);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects to Realtime API Websocket Server
|
||||
* @param {{model?: string}} [settings]
|
||||
* @returns {Promise<true>}
|
||||
*/
|
||||
async connect({ model } = { model: 'gpt-4o-realtime-preview-2024-10-01' }) {
|
||||
if (!this.apiKey && this.url === this.defaultUrl) {
|
||||
console.warn(`No apiKey provided for connection to "${this.url}"`);
|
||||
}
|
||||
if (this.isConnected()) {
|
||||
throw new Error(`Already connected`);
|
||||
}
|
||||
if (globalThis.WebSocket) {
|
||||
/**
|
||||
* Web browser
|
||||
*/
|
||||
if (globalThis.document && this.apiKey) {
|
||||
console.warn(
|
||||
'Warning: Connecting using API key in the browser, this is not recommended',
|
||||
);
|
||||
}
|
||||
const WebSocket = globalThis.WebSocket;
|
||||
const ws = new WebSocket(`${this.url}${model ? `?model=${model}` : ''}`, [
|
||||
'realtime',
|
||||
`openai-insecure-api-key.${this.apiKey}`,
|
||||
'openai-beta.realtime-v1',
|
||||
]);
|
||||
ws.addEventListener('message', (event) => {
|
||||
const message = JSON.parse(event.data);
|
||||
this.receive(message.type, message);
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
const connectionErrorHandler = () => {
|
||||
this.disconnect(ws);
|
||||
reject(new Error(`Could not connect to "${this.url}"`));
|
||||
};
|
||||
ws.addEventListener('error', connectionErrorHandler);
|
||||
ws.addEventListener('open', () => {
|
||||
this.log(`Connected to "${this.url}"`);
|
||||
ws.removeEventListener('error', connectionErrorHandler);
|
||||
ws.addEventListener('error', () => {
|
||||
this.disconnect(ws);
|
||||
this.log(`Error, disconnected from "${this.url}"`);
|
||||
this.dispatch('close', { error: true });
|
||||
});
|
||||
ws.addEventListener('close', () => {
|
||||
this.disconnect(ws);
|
||||
this.log(`Disconnected from "${this.url}"`);
|
||||
this.dispatch('close', { error: false });
|
||||
});
|
||||
this.ws = ws;
|
||||
resolve(true);
|
||||
});
|
||||
});
|
||||
} else {
|
||||
/**
|
||||
* Node.js
|
||||
*/
|
||||
const moduleName = 'ws';
|
||||
const wsModule = await import(/* webpackIgnore: true */ moduleName);
|
||||
const WebSocket = wsModule.default;
|
||||
const ws = new WebSocket(
|
||||
'wss://api.openai.com/v1/realtime?model=gpt-4o-realtime-preview-2024-10-01',
|
||||
[],
|
||||
{
|
||||
finishRequest: (request) => {
|
||||
// Auth
|
||||
request.setHeader('Authorization', `Bearer ${this.apiKey}`);
|
||||
request.setHeader('OpenAI-Beta', 'realtime=v1');
|
||||
request.end();
|
||||
},
|
||||
},
|
||||
);
|
||||
ws.on('message', (data) => {
|
||||
const message = JSON.parse(data.toString());
|
||||
this.receive(message.type, message);
|
||||
});
|
||||
return new Promise((resolve, reject) => {
|
||||
const connectionErrorHandler = () => {
|
||||
this.disconnect(ws);
|
||||
reject(new Error(`Could not connect to "${this.url}"`));
|
||||
};
|
||||
ws.on('error', connectionErrorHandler);
|
||||
ws.on('open', () => {
|
||||
this.log(`Connected to "${this.url}"`);
|
||||
ws.removeListener('error', connectionErrorHandler);
|
||||
ws.on('error', () => {
|
||||
this.disconnect(ws);
|
||||
this.log(`Error, disconnected from "${this.url}"`);
|
||||
this.dispatch('close', { error: true });
|
||||
});
|
||||
ws.on('close', () => {
|
||||
this.disconnect(ws);
|
||||
this.log(`Disconnected from "${this.url}"`);
|
||||
this.dispatch('close', { error: false });
|
||||
});
|
||||
this.ws = ws;
|
||||
resolve(true);
|
||||
});
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnects from Realtime API server
|
||||
* @param {WebSocket} [ws]
|
||||
* @returns {true}
|
||||
*/
|
||||
disconnect(ws) {
|
||||
if (!ws || this.ws === ws) {
|
||||
this.ws && this.ws.close();
|
||||
this.ws = null;
|
||||
return true;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Receives an event from WebSocket and dispatches as "server.{eventName}" and "server.*" events
|
||||
* @param {string} eventName
|
||||
* @param {{[key: string]: any}} event
|
||||
* @returns {true}
|
||||
*/
|
||||
receive(eventName, event) {
|
||||
this.log(`received:`, eventName, event);
|
||||
this.dispatch(`server.${eventName}`, event);
|
||||
this.dispatch('server.*', event);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends an event to WebSocket and dispatches as "client.{eventName}" and "client.*" events
|
||||
* @param {string} eventName
|
||||
* @param {{[key: string]: any}} event
|
||||
* @returns {true}
|
||||
*/
|
||||
send(eventName, data) {
|
||||
if (!this.isConnected()) {
|
||||
throw new Error(`RealtimeAPI is not connected`);
|
||||
}
|
||||
data = data || {};
|
||||
if (typeof data !== 'object') {
|
||||
throw new Error(`data must be an object`);
|
||||
}
|
||||
const event = {
|
||||
event_id: RealtimeUtils.generateId('evt_'),
|
||||
type: eventName,
|
||||
...data,
|
||||
};
|
||||
this.dispatch(`client.${eventName}`, event);
|
||||
this.dispatch('client.*', event);
|
||||
this.log(`sent:`, eventName, event);
|
||||
this.ws.send(JSON.stringify(event));
|
||||
return true;
|
||||
}
|
||||
}
|
||||
676
server-deno/realtime/client.js
Normal file
676
server-deno/realtime/client.js
Normal file
|
|
@ -0,0 +1,676 @@
|
|||
import { RealtimeEventHandler } from './event_handler.js';
|
||||
import { RealtimeAPI } from './api.js';
|
||||
import { RealtimeConversation } from './conversation.js';
|
||||
import { RealtimeUtils } from './utils.js';
|
||||
|
||||
/**
|
||||
* Valid audio formats
|
||||
* @typedef {"pcm16"|"g711_ulaw"|"g711_alaw"} AudioFormatType
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} AudioTranscriptionType
|
||||
* @property {"whisper-1"} model
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} TurnDetectionServerVadType
|
||||
* @property {"server_vad"} type
|
||||
* @property {number} [threshold]
|
||||
* @property {number} [prefix_padding_ms]
|
||||
* @property {number} [silence_duration_ms]
|
||||
*/
|
||||
|
||||
/**
|
||||
* Tool definitions
|
||||
* @typedef {Object} ToolDefinitionType
|
||||
* @property {"function"} [type]
|
||||
* @property {string} name
|
||||
* @property {string} description
|
||||
* @property {{[key: string]: any}} parameters
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SessionResourceType
|
||||
* @property {string} [model]
|
||||
* @property {string[]} [modalities]
|
||||
* @property {string} [instructions]
|
||||
* @property {"alloy"|"ash"|"ballad"|"coral"|"echo"|"sage"|"shimmer"|"verse"} [voice]
|
||||
* @property {AudioFormatType} [input_audio_format]
|
||||
* @property {AudioFormatType} [output_audio_format]
|
||||
* @property {AudioTranscriptionType|null} [input_audio_transcription]
|
||||
* @property {TurnDetectionServerVadType|null} [turn_detection]
|
||||
* @property {ToolDefinitionType[]} [tools]
|
||||
* @property {"auto"|"none"|"required"|{type:"function",name:string}} [tool_choice]
|
||||
* @property {number} [temperature]
|
||||
* @property {number|"inf"} [max_response_output_tokens]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {"in_progress"|"completed"|"incomplete"} ItemStatusType
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} InputTextContentType
|
||||
* @property {"input_text"} type
|
||||
* @property {string} text
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} InputAudioContentType
|
||||
* @property {"input_audio"} type
|
||||
* @property {string} [audio] base64-encoded audio data
|
||||
* @property {string|null} [transcript]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} TextContentType
|
||||
* @property {"text"} type
|
||||
* @property {string} text
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} AudioContentType
|
||||
* @property {"audio"} type
|
||||
* @property {string} [audio] base64-encoded audio data
|
||||
* @property {string|null} [transcript]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} SystemItemType
|
||||
* @property {string|null} [previous_item_id]
|
||||
* @property {"message"} type
|
||||
* @property {ItemStatusType} status
|
||||
* @property {"system"} role
|
||||
* @property {Array<InputTextContentType>} content
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} UserItemType
|
||||
* @property {string|null} [previous_item_id]
|
||||
* @property {"message"} type
|
||||
* @property {ItemStatusType} status
|
||||
* @property {"user"} role
|
||||
* @property {Array<InputTextContentType|InputAudioContentType>} content
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} AssistantItemType
|
||||
* @property {string|null} [previous_item_id]
|
||||
* @property {"message"} type
|
||||
* @property {ItemStatusType} status
|
||||
* @property {"assistant"} role
|
||||
* @property {Array<TextContentType|AudioContentType>} content
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FunctionCallItemType
|
||||
* @property {string|null} [previous_item_id]
|
||||
* @property {"function_call"} type
|
||||
* @property {ItemStatusType} status
|
||||
* @property {string} call_id
|
||||
* @property {string} name
|
||||
* @property {string} arguments
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FunctionCallOutputItemType
|
||||
* @property {string|null} [previous_item_id]
|
||||
* @property {"function_call_output"} type
|
||||
* @property {string} call_id
|
||||
* @property {string} output
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FormattedToolType
|
||||
* @property {"function"} type
|
||||
* @property {string} name
|
||||
* @property {string} call_id
|
||||
* @property {string} arguments
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FormattedPropertyType
|
||||
* @property {Int16Array} [audio]
|
||||
* @property {string} [text]
|
||||
* @property {string} [transcript]
|
||||
* @property {FormattedToolType} [tool]
|
||||
* @property {string} [output]
|
||||
* @property {any} [file]
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FormattedItemType
|
||||
* @property {string} id
|
||||
* @property {string} object
|
||||
* @property {"user"|"assistant"|"system"} [role]
|
||||
* @property {FormattedPropertyType} formatted
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {SystemItemType|UserItemType|AssistantItemType|FunctionCallItemType|FunctionCallOutputItemType} BaseItemType
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {FormattedItemType & BaseItemType} ItemType
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} IncompleteResponseStatusType
|
||||
* @property {"incomplete"} type
|
||||
* @property {"interruption"|"max_output_tokens"|"content_filter"} reason
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} FailedResponseStatusType
|
||||
* @property {"failed"} type
|
||||
* @property {{code: string, message: string}|null} error
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} UsageType
|
||||
* @property {number} total_tokens
|
||||
* @property {number} input_tokens
|
||||
* @property {number} output_tokens
|
||||
*/
|
||||
|
||||
/**
|
||||
* @typedef {Object} ResponseResourceType
|
||||
* @property {"in_progress"|"completed"|"incomplete"|"cancelled"|"failed"} status
|
||||
* @property {IncompleteResponseStatusType|FailedResponseStatusType|null} status_details
|
||||
* @property {ItemType[]} output
|
||||
* @property {UsageType|null} usage
|
||||
*/
|
||||
|
||||
/**
|
||||
* RealtimeClient Class
|
||||
* @class
|
||||
*/
|
||||
export class RealtimeClient extends RealtimeEventHandler {
|
||||
/**
|
||||
* Create a new RealtimeClient instance
|
||||
* @param {{url?: string, apiKey?: string, dangerouslyAllowAPIKeyInBrowser?: boolean, debug?: boolean}} [settings]
|
||||
*/
|
||||
constructor({ url, apiKey, dangerouslyAllowAPIKeyInBrowser, debug } = {}) {
|
||||
super();
|
||||
this.defaultSessionConfig = {
|
||||
modalities: ['text', 'audio'],
|
||||
instructions: '',
|
||||
voice: 'verse',
|
||||
input_audio_format: 'pcm16',
|
||||
output_audio_format: 'pcm16',
|
||||
input_audio_transcription: null,
|
||||
turn_detection: null,
|
||||
tools: [],
|
||||
tool_choice: 'auto',
|
||||
temperature: 0.8,
|
||||
max_response_output_tokens: 4096,
|
||||
};
|
||||
this.sessionConfig = {};
|
||||
this.transcriptionModels = [
|
||||
{
|
||||
model: 'whisper-1',
|
||||
},
|
||||
];
|
||||
this.defaultServerVadConfig = {
|
||||
type: 'server_vad',
|
||||
threshold: 0.5, // 0.0 to 1.0,
|
||||
prefix_padding_ms: 300, // How much audio to include in the audio stream before the speech starts.
|
||||
silence_duration_ms: 200, // How long to wait to mark the speech as stopped.
|
||||
};
|
||||
this.realtime = new RealtimeAPI({
|
||||
url,
|
||||
apiKey,
|
||||
dangerouslyAllowAPIKeyInBrowser,
|
||||
debug,
|
||||
});
|
||||
this.conversation = new RealtimeConversation();
|
||||
this._resetConfig();
|
||||
this._addAPIEventHandlers();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets sessionConfig and conversationConfig to default
|
||||
* @private
|
||||
* @returns {true}
|
||||
*/
|
||||
_resetConfig() {
|
||||
this.sessionCreated = false;
|
||||
this.tools = {};
|
||||
this.sessionConfig = JSON.parse(JSON.stringify(this.defaultSessionConfig));
|
||||
this.inputAudioBuffer = new Int16Array(0);
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sets up event handlers for a fully-functional application control flow
|
||||
* @private
|
||||
* @returns {true}
|
||||
*/
|
||||
_addAPIEventHandlers() {
|
||||
// Event Logging handlers
|
||||
this.realtime.on('client.*', (event) => {
|
||||
const realtimeEvent = {
|
||||
time: new Date().toISOString(),
|
||||
source: 'client',
|
||||
event: event,
|
||||
};
|
||||
this.dispatch('realtime.event', realtimeEvent);
|
||||
});
|
||||
this.realtime.on('server.*', (event) => {
|
||||
const realtimeEvent = {
|
||||
time: new Date().toISOString(),
|
||||
source: 'server',
|
||||
event: event,
|
||||
};
|
||||
this.dispatch('realtime.event', realtimeEvent);
|
||||
});
|
||||
|
||||
// Handles session created event, can optionally wait for it
|
||||
this.realtime.on(
|
||||
'server.session.created',
|
||||
() => (this.sessionCreated = true),
|
||||
);
|
||||
|
||||
// Setup for application control flow
|
||||
const handler = (event, ...args) => {
|
||||
const { item, delta } = this.conversation.processEvent(event, ...args);
|
||||
return { item, delta };
|
||||
};
|
||||
const handlerWithDispatch = (event, ...args) => {
|
||||
const { item, delta } = handler(event, ...args);
|
||||
if (item) {
|
||||
// FIXME: If statement is only here because item.input_audio_transcription.completed
|
||||
// can fire before `item.created`, resulting in empty item.
|
||||
// This happens in VAD mode with empty audio
|
||||
this.dispatch('conversation.updated', { item, delta });
|
||||
}
|
||||
return { item, delta };
|
||||
};
|
||||
const callTool = async (tool) => {
|
||||
try {
|
||||
const jsonArguments = JSON.parse(tool.arguments);
|
||||
const toolConfig = this.tools[tool.name];
|
||||
if (!toolConfig) {
|
||||
throw new Error(`Tool "${tool.name}" has not been added`);
|
||||
}
|
||||
const result = await toolConfig.handler(jsonArguments);
|
||||
this.realtime.send('conversation.item.create', {
|
||||
item: {
|
||||
type: 'function_call_output',
|
||||
call_id: tool.call_id,
|
||||
output: JSON.stringify(result),
|
||||
},
|
||||
});
|
||||
} catch (e) {
|
||||
this.realtime.send('conversation.item.create', {
|
||||
item: {
|
||||
type: 'function_call_output',
|
||||
call_id: tool.call_id,
|
||||
output: JSON.stringify({ error: e.message }),
|
||||
},
|
||||
});
|
||||
}
|
||||
this.createResponse();
|
||||
};
|
||||
|
||||
// Handlers to update internal conversation state
|
||||
this.realtime.on('server.response.created', handler);
|
||||
this.realtime.on('server.response.output_item.added', handler);
|
||||
this.realtime.on('server.response.content_part.added', handler);
|
||||
this.realtime.on('server.input_audio_buffer.speech_started', (event) => {
|
||||
handler(event);
|
||||
this.dispatch('conversation.interrupted');
|
||||
});
|
||||
this.realtime.on(
|
||||
'server.input_audio_buffer.speech_stopped',
|
||||
(event) => handler(event, this.inputAudioBuffer),
|
||||
);
|
||||
|
||||
// Handlers to update application state
|
||||
this.realtime.on('server.conversation.item.created', (event) => {
|
||||
const { item } = handlerWithDispatch(event);
|
||||
this.dispatch('conversation.item.appended', { item });
|
||||
if (item.status === 'completed') {
|
||||
this.dispatch('conversation.item.completed', { item });
|
||||
}
|
||||
});
|
||||
this.realtime.on('server.conversation.item.truncated', handlerWithDispatch);
|
||||
this.realtime.on('server.conversation.item.deleted', handlerWithDispatch);
|
||||
this.realtime.on(
|
||||
'server.conversation.item.input_audio_transcription.completed',
|
||||
handlerWithDispatch,
|
||||
);
|
||||
this.realtime.on(
|
||||
'server.response.audio_transcript.delta',
|
||||
handlerWithDispatch,
|
||||
);
|
||||
this.realtime.on('server.response.audio.delta', handlerWithDispatch);
|
||||
this.realtime.on('server.response.text.delta', handlerWithDispatch);
|
||||
this.realtime.on(
|
||||
'server.response.function_call_arguments.delta',
|
||||
handlerWithDispatch,
|
||||
);
|
||||
this.realtime.on('server.response.output_item.done', async (event) => {
|
||||
const { item } = handlerWithDispatch(event);
|
||||
if (item.status === 'completed') {
|
||||
this.dispatch('conversation.item.completed', { item });
|
||||
}
|
||||
if (item.formatted.tool) {
|
||||
callTool(item.formatted.tool);
|
||||
}
|
||||
});
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Tells us whether the realtime socket is connected and the session has started
|
||||
* @returns {boolean}
|
||||
*/
|
||||
isConnected() {
|
||||
return this.realtime.isConnected();
|
||||
}
|
||||
|
||||
/**
|
||||
* Resets the client instance entirely: disconnects and clears active config
|
||||
* @returns {true}
|
||||
*/
|
||||
reset() {
|
||||
this.disconnect();
|
||||
this.clearEventHandlers();
|
||||
this.realtime.clearEventHandlers();
|
||||
this._resetConfig();
|
||||
this._addAPIEventHandlers();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Connects to the Realtime WebSocket API
|
||||
* Updates session config and conversation config
|
||||
* @returns {Promise<true>}
|
||||
*/
|
||||
async connect({
|
||||
model = 'gpt-4o-mini-realtime-preview-2024-12-17',
|
||||
turn_detection = null,
|
||||
voice,
|
||||
instructions,
|
||||
input_audio_transcription,
|
||||
}) {
|
||||
if (this.isConnected()) {
|
||||
throw new Error(`Already connected, use .disconnect() first`);
|
||||
}
|
||||
await this.realtime.connect({
|
||||
model,
|
||||
});
|
||||
this.updateSession({
|
||||
voice,
|
||||
turn_detection,
|
||||
instructions,
|
||||
input_audio_transcription,
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for a session.created event to be executed before proceeding
|
||||
* @returns {Promise<true>}
|
||||
*/
|
||||
async waitForSessionCreated() {
|
||||
if (!this.isConnected()) {
|
||||
throw new Error(`Not connected, use .connect() first`);
|
||||
}
|
||||
while (!this.sessionCreated) {
|
||||
await new Promise((r) => setTimeout(() => r(), 1));
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Disconnects from the Realtime API and clears the conversation history
|
||||
*/
|
||||
disconnect() {
|
||||
console.log('foobar disconnecting from realtime client');
|
||||
this.sessionCreated = false;
|
||||
this.realtime.isConnected() && this.realtime.disconnect();
|
||||
this.conversation.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Gets the active turn detection mode
|
||||
* @returns {"server_vad"|null}
|
||||
*/
|
||||
getTurnDetectionType() {
|
||||
return this.sessionConfig.turn_detection?.type || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a tool and handler
|
||||
* @param {ToolDefinitionType} definition
|
||||
* @param {function} handler
|
||||
* @returns {{definition: ToolDefinitionType, handler: function}}
|
||||
*/
|
||||
addTool(definition, handler) {
|
||||
if (!definition?.name) {
|
||||
throw new Error(`Missing tool name in definition`);
|
||||
}
|
||||
const name = definition?.name;
|
||||
if (this.tools[name]) {
|
||||
throw new Error(
|
||||
`Tool "${name}" already added. Please use .removeTool("${name}") before trying to add again.`,
|
||||
);
|
||||
}
|
||||
if (typeof handler !== 'function') {
|
||||
throw new Error(`Tool "${name}" handler must be a function`);
|
||||
}
|
||||
this.tools[name] = { definition, handler };
|
||||
this.updateSession();
|
||||
return this.tools[name];
|
||||
}
|
||||
|
||||
/**
|
||||
* Removes a tool
|
||||
* @param {string} name
|
||||
* @returns {true}
|
||||
*/
|
||||
removeTool(name) {
|
||||
if (!this.tools[name]) {
|
||||
throw new Error(`Tool "${name}" does not exist, can not be removed.`);
|
||||
}
|
||||
delete this.tools[name];
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Deletes an item
|
||||
* @param {string} id
|
||||
* @returns {true}
|
||||
*/
|
||||
deleteItem(id) {
|
||||
this.realtime.send('conversation.item.delete', { item_id: id });
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Updates session configuration
|
||||
* If the client is not yet connected, will save details and instantiate upon connection
|
||||
* @param {SessionResourceType} [sessionConfig]
|
||||
*/
|
||||
updateSession({
|
||||
modalities = void 0,
|
||||
instructions = void 0,
|
||||
voice = void 0,
|
||||
input_audio_format = void 0,
|
||||
output_audio_format = void 0,
|
||||
input_audio_transcription = void 0,
|
||||
turn_detection = void 0,
|
||||
tools = void 0,
|
||||
tool_choice = void 0,
|
||||
temperature = void 0,
|
||||
max_response_output_tokens = void 0,
|
||||
} = {}) {
|
||||
modalities !== void 0 && (this.sessionConfig.modalities = modalities);
|
||||
instructions !== void 0 && (this.sessionConfig.instructions = instructions);
|
||||
voice !== void 0 && (this.sessionConfig.voice = voice);
|
||||
input_audio_format !== void 0 &&
|
||||
(this.sessionConfig.input_audio_format = input_audio_format);
|
||||
output_audio_format !== void 0 &&
|
||||
(this.sessionConfig.output_audio_format = output_audio_format);
|
||||
input_audio_transcription !== void 0 &&
|
||||
(this.sessionConfig.input_audio_transcription = input_audio_transcription);
|
||||
turn_detection !== void 0 &&
|
||||
(this.sessionConfig.turn_detection = turn_detection);
|
||||
tools !== void 0 && (this.sessionConfig.tools = tools);
|
||||
tool_choice !== void 0 && (this.sessionConfig.tool_choice = tool_choice);
|
||||
temperature !== void 0 && (this.sessionConfig.temperature = temperature);
|
||||
max_response_output_tokens !== void 0 &&
|
||||
(this.sessionConfig.max_response_output_tokens = max_response_output_tokens);
|
||||
// Load tools from tool definitions + already loaded tools
|
||||
const useTools = [].concat(
|
||||
(tools || []).map((toolDefinition) => {
|
||||
const definition = {
|
||||
type: 'function',
|
||||
...toolDefinition,
|
||||
};
|
||||
if (this.tools[definition?.name]) {
|
||||
throw new Error(
|
||||
`Tool "${definition?.name}" has already been defined`,
|
||||
);
|
||||
}
|
||||
return definition;
|
||||
}),
|
||||
Object.keys(this.tools).map((key) => {
|
||||
return {
|
||||
type: 'function',
|
||||
...this.tools[key].definition,
|
||||
};
|
||||
}),
|
||||
);
|
||||
const session = { ...this.sessionConfig };
|
||||
session.tools = useTools;
|
||||
if (this.realtime.isConnected()) {
|
||||
this.realtime.send('session.update', { session });
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Sends user message content and generates a response
|
||||
* @param {Array<InputTextContentType|InputAudioContentType>} content
|
||||
* @returns {true}
|
||||
*/
|
||||
sendUserMessageContent(content = []) {
|
||||
if (content.length) {
|
||||
for (const c of content) {
|
||||
if (c.type === 'input_audio') {
|
||||
if (c.audio instanceof ArrayBuffer || c.audio instanceof Int16Array) {
|
||||
c.audio = RealtimeUtils.arrayBufferToBase64(c.audio);
|
||||
}
|
||||
}
|
||||
}
|
||||
this.realtime.send('conversation.item.create', {
|
||||
item: {
|
||||
type: 'message',
|
||||
role: 'user',
|
||||
content,
|
||||
},
|
||||
});
|
||||
}
|
||||
this.createResponse();
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Appends user audio to the existing audio buffer
|
||||
* @param {Int16Array|ArrayBuffer} arrayBuffer
|
||||
* @returns {true}
|
||||
*/
|
||||
appendInputAudio(arrayBuffer) {
|
||||
if (arrayBuffer.byteLength > 0) {
|
||||
this.realtime.send('input_audio_buffer.append', {
|
||||
audio: RealtimeUtils.arrayBufferToBase64(arrayBuffer),
|
||||
});
|
||||
this.inputAudioBuffer = RealtimeUtils.mergeInt16Arrays(
|
||||
this.inputAudioBuffer,
|
||||
arrayBuffer,
|
||||
);
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Forces a model response generation
|
||||
* @returns {true}
|
||||
*/
|
||||
createResponse() {
|
||||
if (
|
||||
this.getTurnDetectionType() === null &&
|
||||
this.inputAudioBuffer.byteLength > 0
|
||||
) {
|
||||
this.realtime.send('input_audio_buffer.commit');
|
||||
this.conversation.queueInputAudio(this.inputAudioBuffer);
|
||||
this.inputAudioBuffer = new Int16Array(0);
|
||||
}
|
||||
this.realtime.send('response.create');
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Cancels the ongoing server generation and truncates ongoing generation, if applicable
|
||||
* If no id provided, will simply call `cancel_generation` command
|
||||
* @param {string} id The id of the message to cancel
|
||||
* @param {number} [sampleCount] The number of samples to truncate past for the ongoing generation
|
||||
* @returns {{item: (AssistantItemType | null)}}
|
||||
*/
|
||||
cancelResponse(id, sampleCount = 0) {
|
||||
if (!id) {
|
||||
this.realtime.send('response.cancel');
|
||||
return { item: null };
|
||||
} else if (id) {
|
||||
const item = this.conversation.getItem(id);
|
||||
if (!item) {
|
||||
throw new Error(`Could not find item "${id}"`);
|
||||
}
|
||||
if (item.type !== 'message') {
|
||||
throw new Error(`Can only cancelResponse messages with type "message"`);
|
||||
} else if (item.role !== 'assistant') {
|
||||
throw new Error(
|
||||
`Can only cancelResponse messages with role "assistant"`,
|
||||
);
|
||||
}
|
||||
this.realtime.send('response.cancel');
|
||||
const audioIndex = item.content.findIndex((c) => c.type === 'audio');
|
||||
if (audioIndex === -1) {
|
||||
throw new Error(`Could not find audio on item to cancel`);
|
||||
}
|
||||
this.realtime.send('conversation.item.truncate', {
|
||||
item_id: id,
|
||||
content_index: audioIndex,
|
||||
audio_end_ms: Math.floor(
|
||||
(sampleCount / this.conversation.defaultFrequency) * 1000,
|
||||
),
|
||||
});
|
||||
return { item };
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility for waiting for the next `conversation.item.appended` event to be triggered by the server
|
||||
* @returns {Promise<{item: ItemType}>}
|
||||
*/
|
||||
async waitForNextItem() {
|
||||
const event = await this.waitForNext('conversation.item.appended');
|
||||
const { item } = event;
|
||||
return { item };
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility for waiting for the next `conversation.item.completed` event to be triggered by the server
|
||||
* @returns {Promise<{item: ItemType}>}
|
||||
*/
|
||||
async waitForNextCompletedItem() {
|
||||
const event = await this.waitForNext('conversation.item.completed');
|
||||
const { item } = event;
|
||||
return { item };
|
||||
}
|
||||
}
|
||||
314
server-deno/realtime/conversation.js
Normal file
314
server-deno/realtime/conversation.js
Normal file
|
|
@ -0,0 +1,314 @@
|
|||
import { RealtimeUtils } from './utils.js';
|
||||
|
||||
/**
|
||||
* Contains text and audio information about a item
|
||||
* Can also be used as a delta
|
||||
* @typedef {Object} ItemContentDeltaType
|
||||
* @property {string} [text]
|
||||
* @property {Int16Array} [audio]
|
||||
* @property {string} [arguments]
|
||||
* @property {string} [transcript]
|
||||
*/
|
||||
|
||||
/**
|
||||
* RealtimeConversation holds conversation history
|
||||
* and performs event validation for RealtimeAPI
|
||||
* @class
|
||||
*/
|
||||
export class RealtimeConversation {
|
||||
defaultFrequency = 24_000; // 24,000 Hz
|
||||
|
||||
EventProcessors = {
|
||||
'conversation.item.created': (event) => {
|
||||
const { item } = event;
|
||||
// deep copy values
|
||||
const newItem = JSON.parse(JSON.stringify(item));
|
||||
if (!this.itemLookup[newItem.id]) {
|
||||
this.itemLookup[newItem.id] = newItem;
|
||||
this.items.push(newItem);
|
||||
}
|
||||
newItem.formatted = {};
|
||||
newItem.formatted.audio = new Int16Array(0);
|
||||
newItem.formatted.text = '';
|
||||
newItem.formatted.transcript = '';
|
||||
// If we have a speech item, can populate audio
|
||||
if (this.queuedSpeechItems[newItem.id]) {
|
||||
newItem.formatted.audio = this.queuedSpeechItems[newItem.id].audio;
|
||||
delete this.queuedSpeechItems[newItem.id]; // free up some memory
|
||||
}
|
||||
// Populate formatted text if it comes out on creation
|
||||
if (newItem.content) {
|
||||
const textContent = newItem.content.filter((c) =>
|
||||
['text', 'input_text'].includes(c.type)
|
||||
);
|
||||
for (const content of textContent) {
|
||||
newItem.formatted.text += content.text;
|
||||
}
|
||||
}
|
||||
// If we have a transcript item, can pre-populate transcript
|
||||
if (this.queuedTranscriptItems[newItem.id]) {
|
||||
newItem.formatted.transcript = this.queuedTranscriptItems.transcript;
|
||||
delete this.queuedTranscriptItems[newItem.id];
|
||||
}
|
||||
if (newItem.type === 'message') {
|
||||
if (newItem.role === 'user') {
|
||||
newItem.status = 'completed';
|
||||
if (this.queuedInputAudio) {
|
||||
newItem.formatted.audio = this.queuedInputAudio;
|
||||
this.queuedInputAudio = null;
|
||||
}
|
||||
} else {
|
||||
newItem.status = 'in_progress';
|
||||
}
|
||||
} else if (newItem.type === 'function_call') {
|
||||
newItem.formatted.tool = {
|
||||
type: 'function',
|
||||
name: newItem.name,
|
||||
call_id: newItem.call_id,
|
||||
arguments: '',
|
||||
};
|
||||
newItem.status = 'in_progress';
|
||||
} else if (newItem.type === 'function_call_output') {
|
||||
newItem.status = 'completed';
|
||||
newItem.formatted.output = newItem.output;
|
||||
}
|
||||
return { item: newItem, delta: null };
|
||||
},
|
||||
'conversation.item.truncated': (event) => {
|
||||
const { item_id, audio_end_ms } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(`item.truncated: Item "${item_id}" not found`);
|
||||
}
|
||||
const endIndex = Math.floor(
|
||||
(audio_end_ms * this.defaultFrequency) / 1000,
|
||||
);
|
||||
item.formatted.transcript = '';
|
||||
item.formatted.audio = item.formatted.audio.slice(0, endIndex);
|
||||
return { item, delta: null };
|
||||
},
|
||||
'conversation.item.deleted': (event) => {
|
||||
const { item_id } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(`item.deleted: Item "${item_id}" not found`);
|
||||
}
|
||||
delete this.itemLookup[item.id];
|
||||
const index = this.items.indexOf(item);
|
||||
if (index > -1) {
|
||||
this.items.splice(index, 1);
|
||||
}
|
||||
return { item, delta: null };
|
||||
},
|
||||
'conversation.item.input_audio_transcription.completed': (event) => {
|
||||
const { item_id, content_index, transcript } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
// We use a single space to represent an empty transcript for .formatted values
|
||||
// Otherwise it looks like no transcript provided
|
||||
const formattedTranscript = transcript || ' ';
|
||||
if (!item) {
|
||||
// We can receive transcripts in VAD mode before item.created
|
||||
// This happens specifically when audio is empty
|
||||
this.queuedTranscriptItems[item_id] = {
|
||||
transcript: formattedTranscript,
|
||||
};
|
||||
return { item: null, delta: null };
|
||||
} else {
|
||||
item.content[content_index].transcript = transcript;
|
||||
item.formatted.transcript = formattedTranscript;
|
||||
return { item, delta: { transcript } };
|
||||
}
|
||||
},
|
||||
'input_audio_buffer.speech_started': (event) => {
|
||||
const { item_id, audio_start_ms } = event;
|
||||
this.queuedSpeechItems[item_id] = { audio_start_ms };
|
||||
return { item: null, delta: null };
|
||||
},
|
||||
'input_audio_buffer.speech_stopped': (event, inputAudioBuffer) => {
|
||||
const { item_id, audio_end_ms } = event;
|
||||
if (!this.queuedSpeechItems[item_id]) {
|
||||
this.queuedSpeechItems[item_id] = { audio_start_ms: audio_end_ms };
|
||||
}
|
||||
const speech = this.queuedSpeechItems[item_id];
|
||||
speech.audio_end_ms = audio_end_ms;
|
||||
if (inputAudioBuffer) {
|
||||
const startIndex = Math.floor(
|
||||
(speech.audio_start_ms * this.defaultFrequency) / 1000,
|
||||
);
|
||||
const endIndex = Math.floor(
|
||||
(speech.audio_end_ms * this.defaultFrequency) / 1000,
|
||||
);
|
||||
speech.audio = inputAudioBuffer.slice(startIndex, endIndex);
|
||||
}
|
||||
return { item: null, delta: null };
|
||||
},
|
||||
'response.created': (event) => {
|
||||
const { response } = event;
|
||||
if (!this.responseLookup[response.id]) {
|
||||
this.responseLookup[response.id] = response;
|
||||
this.responses.push(response);
|
||||
}
|
||||
return { item: null, delta: null };
|
||||
},
|
||||
'response.output_item.added': (event) => {
|
||||
const { response_id, item } = event;
|
||||
const response = this.responseLookup[response_id];
|
||||
if (!response) {
|
||||
throw new Error(
|
||||
`response.output_item.added: Response "${response_id}" not found`,
|
||||
);
|
||||
}
|
||||
response.output.push(item.id);
|
||||
return { item: null, delta: null };
|
||||
},
|
||||
'response.output_item.done': (event) => {
|
||||
const { item } = event;
|
||||
if (!item) {
|
||||
throw new Error(`response.output_item.done: Missing "item"`);
|
||||
}
|
||||
const foundItem = this.itemLookup[item.id];
|
||||
if (!foundItem) {
|
||||
throw new Error(
|
||||
`response.output_item.done: Item "${item.id}" not found`,
|
||||
);
|
||||
}
|
||||
foundItem.status = item.status;
|
||||
return { item: foundItem, delta: null };
|
||||
},
|
||||
'response.content_part.added': (event) => {
|
||||
const { item_id, part } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(
|
||||
`response.content_part.added: Item "${item_id}" not found`,
|
||||
);
|
||||
}
|
||||
item.content.push(part);
|
||||
return { item, delta: null };
|
||||
},
|
||||
'response.audio_transcript.delta': (event) => {
|
||||
const { item_id, content_index, delta } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(
|
||||
`response.audio_transcript.delta: Item "${item_id}" not found`,
|
||||
);
|
||||
}
|
||||
item.content[content_index].transcript += delta;
|
||||
item.formatted.transcript += delta;
|
||||
return { item, delta: { transcript: delta } };
|
||||
},
|
||||
'response.audio.delta': (event) => {
|
||||
const { item_id, content_index, delta } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(`response.audio.delta: Item "${item_id}" not found`);
|
||||
}
|
||||
// This never gets renderered, we care about the file data instead
|
||||
// item.content[content_index].audio += delta;
|
||||
const arrayBuffer = RealtimeUtils.base64ToArrayBuffer(delta);
|
||||
const appendValues = new Int16Array(arrayBuffer);
|
||||
item.formatted.audio = RealtimeUtils.mergeInt16Arrays(
|
||||
item.formatted.audio,
|
||||
appendValues,
|
||||
);
|
||||
return { item, delta: { audio: appendValues } };
|
||||
},
|
||||
'response.text.delta': (event) => {
|
||||
const { item_id, content_index, delta } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(`response.text.delta: Item "${item_id}" not found`);
|
||||
}
|
||||
item.content[content_index].text += delta;
|
||||
item.formatted.text += delta;
|
||||
return { item, delta: { text: delta } };
|
||||
},
|
||||
'response.function_call_arguments.delta': (event) => {
|
||||
const { item_id, delta } = event;
|
||||
const item = this.itemLookup[item_id];
|
||||
if (!item) {
|
||||
throw new Error(
|
||||
`response.function_call_arguments.delta: Item "${item_id}" not found`,
|
||||
);
|
||||
}
|
||||
item.arguments += delta;
|
||||
item.formatted.tool.arguments += delta;
|
||||
return { item, delta: { arguments: delta } };
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* Create a new RealtimeConversation instance
|
||||
* @returns {RealtimeConversation}
|
||||
*/
|
||||
constructor() {
|
||||
this.clear();
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears the conversation history and resets to default
|
||||
* @returns {true}
|
||||
*/
|
||||
clear() {
|
||||
this.itemLookup = {};
|
||||
this.items = [];
|
||||
this.responseLookup = {};
|
||||
this.responses = [];
|
||||
this.queuedSpeechItems = {};
|
||||
this.queuedTranscriptItems = {};
|
||||
this.queuedInputAudio = null;
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue input audio for manual speech event
|
||||
* @param {Int16Array} inputAudio
|
||||
* @returns {Int16Array}
|
||||
*/
|
||||
queueInputAudio(inputAudio) {
|
||||
this.queuedInputAudio = inputAudio;
|
||||
return inputAudio;
|
||||
}
|
||||
|
||||
/**
|
||||
* Process an event from the WebSocket server and compose items
|
||||
* @param {Object} event
|
||||
* @param {...any} args
|
||||
* @returns {item: import('./client.js').ItemType | null, delta: ItemContentDeltaType | null}
|
||||
*/
|
||||
processEvent(event, ...args) {
|
||||
if (!event.event_id) {
|
||||
console.error(event);
|
||||
throw new Error(`Missing "event_id" on event`);
|
||||
}
|
||||
if (!event.type) {
|
||||
console.error(event);
|
||||
throw new Error(`Missing "type" on event`);
|
||||
}
|
||||
const eventProcessor = this.EventProcessors[event.type];
|
||||
if (!eventProcessor) {
|
||||
throw new Error(
|
||||
`Missing conversation event processor for "${event.type}"`,
|
||||
);
|
||||
}
|
||||
return eventProcessor.call(this, event, ...args);
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves a item by id
|
||||
* @param {string} id
|
||||
* @returns {import('./client.js').ItemType}
|
||||
*/
|
||||
getItem(id) {
|
||||
return this.itemLookup[id] || null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Retrieves all items in the conversation
|
||||
* @returns {import('./client.js').ItemType[]}
|
||||
*/
|
||||
getItems() {
|
||||
return this.items.slice();
|
||||
}
|
||||
}
|
||||
143
server-deno/realtime/event_handler.js
Normal file
143
server-deno/realtime/event_handler.js
Normal file
|
|
@ -0,0 +1,143 @@
|
|||
/**
|
||||
* EventHandler callback
|
||||
* @typedef {(event: {[key: string]: any}): void} EventHandlerCallbackType
|
||||
*/
|
||||
|
||||
const sleep = (t) => new Promise((r) => setTimeout(() => r(), t));
|
||||
|
||||
/**
|
||||
* Inherited class for RealtimeAPI and RealtimeClient
|
||||
* Adds basic event handling
|
||||
* @class
|
||||
*/
|
||||
export class RealtimeEventHandler {
|
||||
/**
|
||||
* Create a new RealtimeEventHandler instance
|
||||
* @returns {RealtimeEventHandler}
|
||||
*/
|
||||
constructor() {
|
||||
this.eventHandlers = {};
|
||||
this.nextEventHandlers = {};
|
||||
}
|
||||
|
||||
/**
|
||||
* Clears all event handlers
|
||||
* @returns {true}
|
||||
*/
|
||||
clearEventHandlers() {
|
||||
this.eventHandlers = {};
|
||||
this.nextEventHandlers = {};
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen to specific events
|
||||
* @param {string} eventName The name of the event to listen to
|
||||
* @param {EventHandlerCallbackType} callback Code to execute on event
|
||||
* @returns {EventHandlerCallbackType}
|
||||
*/
|
||||
on(eventName, callback) {
|
||||
this.eventHandlers[eventName] = this.eventHandlers[eventName] || [];
|
||||
this.eventHandlers[eventName].push(callback);
|
||||
return callback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Listen for the next event of a specified type
|
||||
* @param {string} eventName The name of the event to listen to
|
||||
* @param {EventHandlerCallbackType} callback Code to execute on event
|
||||
* @returns {EventHandlerCallbackType}
|
||||
*/
|
||||
onNext(eventName, callback) {
|
||||
this.nextEventHandlers[eventName] = this.nextEventHandlers[eventName] || [];
|
||||
this.nextEventHandlers[eventName].push(callback);
|
||||
return callback;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns off event listening for specific events
|
||||
* Calling without a callback will remove all listeners for the event
|
||||
* @param {string} eventName
|
||||
* @param {EventHandlerCallbackType} [callback]
|
||||
* @returns {true}
|
||||
*/
|
||||
off(eventName, callback) {
|
||||
const handlers = this.eventHandlers[eventName] || [];
|
||||
if (callback) {
|
||||
const index = handlers.indexOf(callback);
|
||||
if (index === -1) {
|
||||
throw new Error(
|
||||
`Could not turn off specified event listener for "${eventName}": not found as a listener`,
|
||||
);
|
||||
}
|
||||
handlers.splice(index, 1);
|
||||
} else {
|
||||
delete this.eventHandlers[eventName];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Turns off event listening for the next event of a specific type
|
||||
* Calling without a callback will remove all listeners for the next event
|
||||
* @param {string} eventName
|
||||
* @param {EventHandlerCallbackType} [callback]
|
||||
* @returns {true}
|
||||
*/
|
||||
offNext(eventName, callback) {
|
||||
const nextHandlers = this.nextEventHandlers[eventName] || [];
|
||||
if (callback) {
|
||||
const index = nextHandlers.indexOf(callback);
|
||||
if (index === -1) {
|
||||
throw new Error(
|
||||
`Could not turn off specified next event listener for "${eventName}": not found as a listener`,
|
||||
);
|
||||
}
|
||||
nextHandlers.splice(index, 1);
|
||||
} else {
|
||||
delete this.nextEventHandlers[eventName];
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Waits for next event of a specific type and returns the payload
|
||||
* @param {string} eventName
|
||||
* @param {number|null} [timeout]
|
||||
* @returns {Promise<{[key: string]: any}|null>}
|
||||
*/
|
||||
async waitForNext(eventName, timeout = null) {
|
||||
const t0 = Date.now();
|
||||
let nextEvent;
|
||||
this.onNext(eventName, (event) => (nextEvent = event));
|
||||
while (!nextEvent) {
|
||||
if (timeout) {
|
||||
const t1 = Date.now();
|
||||
if (t1 - t0 > timeout) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
await sleep(1);
|
||||
}
|
||||
return nextEvent;
|
||||
}
|
||||
|
||||
/**
|
||||
* Executes all events in the order they were added, with .on() event handlers executing before .onNext() handlers
|
||||
* @param {string} eventName
|
||||
* @param {any} event
|
||||
* @returns {true}
|
||||
*/
|
||||
dispatch(eventName, event) {
|
||||
const handlers = [].concat(this.eventHandlers[eventName] || []);
|
||||
for (const handler of handlers) {
|
||||
handler(event);
|
||||
}
|
||||
const nextHandlers = [].concat(this.nextEventHandlers[eventName] || []);
|
||||
for (const nextHandler of nextHandlers) {
|
||||
nextHandler(event);
|
||||
}
|
||||
delete this.nextEventHandlers[eventName];
|
||||
return true;
|
||||
}
|
||||
}
|
||||
102
server-deno/realtime/utils.js
Normal file
102
server-deno/realtime/utils.js
Normal file
|
|
@ -0,0 +1,102 @@
|
|||
const atob = globalThis.atob;
|
||||
const btoa = globalThis.btoa;
|
||||
|
||||
/**
|
||||
* Basic utilities for the RealtimeAPI
|
||||
* @class
|
||||
*/
|
||||
export class RealtimeUtils {
|
||||
/**
|
||||
* Converts Float32Array of amplitude data to ArrayBuffer in Int16Array format
|
||||
* @param {Float32Array} float32Array
|
||||
* @returns {ArrayBuffer}
|
||||
*/
|
||||
static floatTo16BitPCM(float32Array) {
|
||||
const buffer = new ArrayBuffer(float32Array.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
let offset = 0;
|
||||
for (let i = 0; i < float32Array.length; i++, offset += 2) {
|
||||
let s = Math.max(-1, Math.min(1, float32Array[i]));
|
||||
view.setInt16(offset, s < 0 ? s * 0x8000 : s * 0x7fff, true);
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts a base64 string to an ArrayBuffer
|
||||
* @param {string} base64
|
||||
* @returns {ArrayBuffer}
|
||||
*/
|
||||
static base64ToArrayBuffer(base64) {
|
||||
const binaryString = atob(base64);
|
||||
const len = binaryString.length;
|
||||
const bytes = new Uint8Array(len);
|
||||
for (let i = 0; i < len; i++) {
|
||||
bytes[i] = binaryString.charCodeAt(i);
|
||||
}
|
||||
return bytes.buffer;
|
||||
}
|
||||
|
||||
/**
|
||||
* Converts an ArrayBuffer, Int16Array or Float32Array to a base64 string
|
||||
* @param {ArrayBuffer|Int16Array|Float32Array} arrayBuffer
|
||||
* @returns {string}
|
||||
*/
|
||||
static arrayBufferToBase64(arrayBuffer) {
|
||||
if (arrayBuffer instanceof Float32Array) {
|
||||
arrayBuffer = this.floatTo16BitPCM(arrayBuffer);
|
||||
} else if (arrayBuffer instanceof Int16Array) {
|
||||
arrayBuffer = arrayBuffer.buffer;
|
||||
}
|
||||
let binary = '';
|
||||
let bytes = new Uint8Array(arrayBuffer);
|
||||
const chunkSize = 0x8000; // 32KB chunk size
|
||||
for (let i = 0; i < bytes.length; i += chunkSize) {
|
||||
let chunk = bytes.subarray(i, i + chunkSize);
|
||||
binary += String.fromCharCode.apply(null, chunk);
|
||||
}
|
||||
return btoa(binary);
|
||||
}
|
||||
|
||||
/**
|
||||
* Merge two Int16Arrays from Int16Arrays or ArrayBuffers
|
||||
* @param {ArrayBuffer|Int16Array} left
|
||||
* @param {ArrayBuffer|Int16Array} right
|
||||
* @returns {Int16Array}
|
||||
*/
|
||||
static mergeInt16Arrays(left, right) {
|
||||
if (left instanceof ArrayBuffer) {
|
||||
left = new Int16Array(left);
|
||||
}
|
||||
if (right instanceof ArrayBuffer) {
|
||||
right = new Int16Array(right);
|
||||
}
|
||||
if (!(left instanceof Int16Array) || !(right instanceof Int16Array)) {
|
||||
throw new Error(`Both items must be Int16Array`);
|
||||
}
|
||||
const newValues = new Int16Array(left.length + right.length);
|
||||
for (let i = 0; i < left.length; i++) {
|
||||
newValues[i] = left[i];
|
||||
}
|
||||
for (let j = 0; j < right.length; j++) {
|
||||
newValues[left.length + j] = right[j];
|
||||
}
|
||||
return newValues;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generates an id to send with events and messages
|
||||
* @param {string} prefix
|
||||
* @param {number} [length]
|
||||
* @returns {string}
|
||||
*/
|
||||
static generateId(prefix, length = 21) {
|
||||
// base58; non-repeating chars
|
||||
const chars = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
|
||||
const str = Array(length - prefix.length)
|
||||
.fill(0)
|
||||
.map((_) => chars[Math.floor(Math.random() * chars.length)])
|
||||
.join('');
|
||||
return `${prefix}${str}`;
|
||||
}
|
||||
}
|
||||
Loading…
Reference in a new issue