AIvoices/server-deno/models/gemini.ts

import { Buffer } from "node:buffer";
import type { WebSocketServer as _WebSocketServer } from "npm:@types/ws";
import {
    EndSensitivity,
    GoogleGenAI,
    LiveConnectConfig,
    LiveServerMessage,
    Modality,
    Session,
} from "npm:@google/genai";
import { createOpusPacketizer, geminiApiKey, isDev, defaultGeminiVoice } from "../utils.ts";
import { addConversation } from "../supabase.ts";

export const connectToGemini = async ({
    ws,
    payload,
    connectionPcmFile,
    firstMessage,
    systemPrompt,
    closeHandler,
}: ProviderArgs) => {
    const { user, supabase } = payload;
    const voiceName = user.personality?.oai_voice ?? defaultGeminiVoice;

    const opus = createOpusPacketizer((packet) => ws.send(packet));

    console.log(`Connecting with Gemini key "${geminiApiKey?.slice(0, 3)}..."`);

    // Initialize Google GenAI
    const ai = new GoogleGenAI({ apiKey: geminiApiKey });
    const model = "gemini-2.5-flash-native-audio-preview-09-2025";
    const config: LiveConnectConfig = {
        responseModalities: [Modality.AUDIO],
        systemInstruction: systemPrompt,
        speechConfig: {
            voiceConfig: {
                prebuiltVoiceConfig: {
                    voiceName: voiceName,
                },
            },
        },
        realtimeInputConfig: {
            automaticActivityDetection: {
                disabled: false, // Keep VAD enabled
                endOfSpeechSensitivity: EndSensitivity.END_SENSITIVITY_LOW,
                silenceDurationMs: 100,
            },
        },
        outputAudioTranscription: {},
        inputAudioTranscription: {},
    };

    // Response queue for handling Google's callback-based responses
    const responseQueue: LiveServerMessage[] = [];
    let geminiSession: Session | null = null;

    async function waitMessage() {
        let done = false;
        let message: LiveServerMessage | undefined = undefined;
        while (!done) {
            message = responseQueue.shift();
            if (message) {
                done = true;
            } else {
                await new Promise((resolve) => setTimeout(resolve, 10));
            }
        }
        return message;
    }

    async function handleTurn() {
        const turns: any[] = [];
        let done = false;
        while (!done) {
            const message = await waitMessage();
            turns.push(message);

            if (message.serverContent) {
                if (message.serverContent.generationComplete) {
                    opus.reset();
                    ws.send(JSON.stringify({
                        type: "server",
                        msg: "RESPONSE.CREATED",
                    }));
                    done = true;
                }
            }
        }
        return turns;
    }

    async function processGeminiTurns() {
        try {
            console.log("Processing Gemini turns");
            while (geminiSession) {
                const turns = await handleTurn();

                // Combine all audio data from this turn
                const combinedAudio = turns.reduce(
                    (acc: number[], turn: any) => {
                        if (turn.data) {
                            const buffer = Buffer.from(turn.data, "base64");
                            const intArray = new Int16Array(
                                buffer.buffer,
                                buffer.byteOffset,
                                buffer.byteLength /
                                    Int16Array.BYTES_PER_ELEMENT,
                            );
                            return acc.concat(Array.from(intArray));
                        }
                        return acc;
                    },
                    [],
                );

                if (combinedAudio.length > 0) {
                    // Convert back to buffer and send to client
                    const audioBuffer = new Int16Array(combinedAudio);
                    const buffer = Buffer.from(audioBuffer.buffer);

                    // Use Opus packetizer to encode and send audio
                    opus.push(buffer);
                    opus.flush(true);
                }

                // Handle text responses if any
                let outputTranscriptionText = "";
                let inputTranscriptionText = "";
                for (const turn of turns as LiveServerMessage[]) {
                    if (
                        turn.serverContent &&
                        turn.serverContent.outputTranscription
                    ) {
                        outputTranscriptionText +=
                            turn.serverContent.outputTranscription.text;
                    }

                    if (
                        turn.serverContent &&
                        turn.serverContent.inputTranscription
                    ) {
                        inputTranscriptionText +=
                            turn.serverContent.inputTranscription.text;
                    }
                }

                // Send completion signal
                ws.send(JSON.stringify({
                    type: "server",
                    msg: "RESPONSE.COMPLETE",
                }));

                // Add user transcription to supabase
                await addConversation(
                    supabase,
                    "user",
                    inputTranscriptionText,
                    user,
                );

                // Add assistant transcription to supabase
                await addConversation(
                    supabase,
                    "assistant",
                    outputTranscriptionText,
                    user,
                );
            }
        } catch (error) {
            console.error("Error processing Gemini turns:", error);
        }
    }

    // Connect to Google Gemini Live
    try {
        geminiSession = await ai.live.connect({
            model: model,
            callbacks: {
                onopen: function () {
                    console.log("Gemini session opened");
                },
                onmessage: function (message: LiveServerMessage) {
                    responseQueue.push(message);
                },
                onerror: function (e: any) {
                    console.error("Gemini error:", e.message);
                    ws.send(
                        JSON.stringify({
                            type: "server",
                            msg: "RESPONSE.ERROR",
                        }),
                    );
                },
                onclose: function (e: any) {
                    console.log("Gemini session closed:", e.reason);
                },
            },
            config: config,
        });

        console.log("Connected to Gemini successfully!");
        // Send first message if available
        const inputTurns = [{
            role: "user",
            parts: [{ text: firstMessage }],
        }];
        geminiSession?.sendClientContent({ turns: inputTurns });
        processGeminiTurns();
    } catch (e: unknown) {
        console.log(`Error connecting to Gemini: ${e}`);
        ws.close();
        return;
    }

    ws.on("message", (data: any, isBinary: boolean) => {
        try {
            if (isBinary) {
                // Handle binary audio data from ESP32
                const base64Data = data.toString("base64");

                if (isDev && connectionPcmFile) {
                    connectionPcmFile.write(data);
                }

                // Send audio to Gemini
                geminiSession?.sendRealtimeInput({
                    audio: {
                        data: base64Data,
                        mimeType: "audio/pcm;rate=24000",
                    },
                });
            }
        } catch (e: unknown) {
            console.error("Error handling message:", (e as Error).message);
        }
    });

    ws.on("error", (error: any) => {
        console.error("WebSocket error:", error);
        geminiSession?.close();
    });

    ws.on("close", async (code: number, reason: string) => {
        console.log(`WebSocket closed with code ${code}, reason: ${reason}`);
        await closeHandler();
        opus.close();
        geminiSession?.close();
        if (isDev && connectionPcmFile) {
            connectionPcmFile.close();
            console.log("Closed debug audio file.");
        }
    });
};