From f6190eb2117754bb5e973e7c63099b853f5c362f Mon Sep 17 00:00:00 2001
From: akdeb <deb2@illinois.edu>
Date: Wed, 4 Jun 2025 19:31:36 +0100
Subject: [PATCH] adding first message instuctions

---
 firmware-arduino/src/Config.h                 |   4 +--
 .../CreateCharacter/BuildDashboard.tsx        |  29 +++++++++++++-----
 .../Playground/ModifyCharacterSheet.tsx       |   6 ++++
 .../app/components/Realtime/App.tsx           |   8 ++++-
 ...armoon_default.jpeg => elato_default.jpeg} | Bin
 frontend-nextjs/types/types.d.ts              |   1 +
 server-deno/main.ts                           |   3 +-
 server-deno/supabase.ts                       |  23 +++++++++-----
 server-deno/types.d.ts                        |   1 +
 .../20250604180849_add_first_message.sql      |   2 ++
 supabase/seed.sql                             |   2 +-
 11 files changed, 60 insertions(+), 19 deletions(-)
 rename frontend-nextjs/public/personality/{starmoon_default.jpeg => elato_default.jpeg} (100%)
 create mode 100644 supabase/migrations/20250604180849_add_first_message.sql

diff --git a/firmware-arduino/src/Config.h b/firmware-arduino/src/Config.h
index df66cbf..e4980e5 100644
--- a/firmware-arduino/src/Config.h
+++ b/firmware-arduino/src/Config.h
@@ -13,9 +13,9 @@
 // Pick one of the following (DEV_MODE, PROD_MODE, ELATO_MODE) , comment the rest
 // For ELATO_MODE, you will need to register your DIY Hardware on the Elato website
 
-// #define DEV_MODE
+#define DEV_MODE
 // #define PROD_MODE
-#define ELATO_MODE
+// #define ELATO_MODE
 
 
 // ---------- Touch mode ----------
diff --git a/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx b/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx
index 9c0febf..175527c 100644
--- a/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx
+++ b/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx
@@ -29,6 +29,7 @@ const formSchema = z.object({
   title: z.string().min(2, "Minimum 2 characters").max(50, "Maximum 50 characters"),
   description: z.string().min(50, "Minimum 50 characters").max(200, "Maximum 200 characters"),
   prompt: z.string().min(100, "Minimum 100 characters").max(1000, "Maximum 1000 characters"),
+  firstMessagePrompt: z.string().min(50, "Minimum 50 characters").max(150, "Maximum 150 characters"),
   voice: z.string().min(1, "Voice selection is required"),
   voiceCharacteristics: z.object({
     features: z.string().min(10, "Minimum 10 characters").max(150, "Maximum 150 characters"),
@@ -41,22 +42,17 @@ type FormData = z.infer<typeof formSchema>;
 
 const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
     selectedUser,
-    allLanguages,
 }) => {
     const supabase = createClient();
     const router = useRouter();
     const [currentStep, setCurrentStep] = useState<'personality' | 'voice'>('personality');
     const [isSubmitting, setIsSubmitting] = useState(false);
 
-
-    const [languageState, setLanguageState] = useState<string>(
-        selectedUser.language_code! // Initial value from props
-    );
-
     const [formData, setFormData] = useState({
         title: '',
         description: '',
         prompt: '',
+        firstMessagePrompt: '',
         voice: '',
         voiceCharacteristics: {
           features: '',
@@ -182,7 +178,8 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
         key: formData.title.toLowerCase().replace(/ /g, '_') + "_" + uuidv4(),
         creator_id: selectedUser.user_id,
         short_description: formData.description,
-        pitch_factor: formData.voiceCharacteristics.pitchFactor
+        pitch_factor: formData.voiceCharacteristics.pitchFactor,
+        first_message_prompt: formData.firstMessagePrompt
       });
 
       if (personality) {
@@ -257,6 +254,7 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
 
     return (
       <div className="overflow-hidden pb-2 w-full flex-auto flex flex-col pl-1 max-w-screen-sm">
+      <Heading />
       <form onSubmit={handleSubmit} className="space-y-6 mt-8 w-full pr-1">
     
       {currentStep === 'personality' ? 
@@ -357,6 +355,23 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
  <span className="text-gray-500">{formData.prompt.length}/1000</span>
  </p>
  </div>
+ <div className="space-y-2">
+ <Label htmlFor="firstMessagePrompt">First message prompt</Label>
+ <Textarea 
+ id="firstMessagePrompt"
+ placeholder="How your AI character should respond in the first message to the user..." 
+ rows={4}
+ value={formData.firstMessagePrompt}
+ onChange={(e) => handleInputChange('firstMessagePrompt', e.target.value)}
+ onBlur={() => handleBlur('firstMessagePrompt')} 
+          />
+ <p className="text-sm flex justify-between">
+ <span className={formErrors.firstMessagePrompt ? "text-red-500" : "text-gray-500"}>
+ {formErrors.firstMessagePrompt}
+ </span>
+ <span className="text-gray-500">{formData.firstMessagePrompt.length}/150</span>
+ </p>
+ </div>
  </div> :
         <div className="space-y-6">                      
         {/* Pitch Slider */}
diff --git a/frontend-nextjs/app/components/Playground/ModifyCharacterSheet.tsx b/frontend-nextjs/app/components/Playground/ModifyCharacterSheet.tsx
index c22bbc7..0bf3ed7 100644
--- a/frontend-nextjs/app/components/Playground/ModifyCharacterSheet.tsx
+++ b/frontend-nextjs/app/components/Playground/ModifyCharacterSheet.tsx
@@ -73,6 +73,12 @@ const ModifyCharacterSheet: React.FC<ModifyCharacterSheetProps> = ({
                     <p className="text-gray-600 whitespace-pre-line">
                             {openPersonality.character_prompt}
                         </p>
+                        <p className="text-gray-400">
+                        {"First message prompt"}
+                    </p>
+                    <p className="text-gray-600">
+                            {openPersonality.first_message_prompt}
+                        </p>
                      <p className="text-gray-400">
                         {"Voice prompt"}
                     </p>
diff --git a/frontend-nextjs/app/components/Realtime/App.tsx b/frontend-nextjs/app/components/Realtime/App.tsx
index 519086b..02df44b 100644
--- a/frontend-nextjs/app/components/Realtime/App.tsx
+++ b/frontend-nextjs/app/components/Realtime/App.tsx
@@ -210,6 +210,12 @@ function App({ personalityIdState, isDoctor, userId }: AppProps) {
     );
   };
 
+  const createFirstMessage = () => {
+    return personality?.first_message_prompt
+    ? `Always start the conversation following these instructions from the user: ${personality?.first_message_prompt}`
+    : "The user is initiating a new chat here. Say something!";
+  }
+
   const updateSession = (shouldTriggerResponse: boolean = false) => {
     sendClientEvent(
       { type: "input_audio_buffer.clear" },
@@ -247,7 +253,7 @@ function App({ personalityIdState, isDoctor, userId }: AppProps) {
     sendClientEvent(sessionUpdateEvent);
 
     if (shouldTriggerResponse) {
-      sendSimulatedUserMessage(isDoctor ? "Ask the doctor if everything is good and how you can help them and their patient." : "The user is initiating a new chat here. Say something!");
+      sendSimulatedUserMessage(isDoctor ? "Ask the doctor if everything is good and how you can help them and their patient." : createFirstMessage());
     }
   };
 
diff --git a/frontend-nextjs/public/personality/starmoon_default.jpeg b/frontend-nextjs/public/personality/elato_default.jpeg
similarity index 100%
rename from frontend-nextjs/public/personality/starmoon_default.jpeg
rename to frontend-nextjs/public/personality/elato_default.jpeg
diff --git a/frontend-nextjs/types/types.d.ts b/frontend-nextjs/types/types.d.ts
index 793451f..7808879 100644
--- a/frontend-nextjs/types/types.d.ts
+++ b/frontend-nextjs/types/types.d.ts
@@ -123,6 +123,7 @@ declare global {
         short_description: string;
         character_prompt: string;
         voice_prompt: string;
+        first_message_prompt: string;
         creator_id: string | null;
         pitch_factor: number;
     }
diff --git a/server-deno/main.ts b/server-deno/main.ts
index 5535c09..42f3dab 100644
--- a/server-deno/main.ts
+++ b/server-deno/main.ts
@@ -132,6 +132,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
         isDoctor,
     );
     const firstMessage = createFirstMessage(chatHistory, payload);
+    console.log("firstMessage", firstMessage);
     const systemPrompt = createSystemPrompt(chatHistory, payload);
     let sessionStartTime: number;
     let currentItemId: string | null = null;
@@ -147,7 +148,7 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
         if (event.type === "session.created") {
             console.log("session created", event);
             sessionStartTime = Date.now();
-            sendFirstMessage(client, firstMessage ?? "");
+            sendFirstMessage(client, firstMessage);
         } else if (event.type === "session.updated") {
             console.log("session updated", event);
         } else if (event.type === "error") {
diff --git a/server-deno/supabase.ts b/server-deno/supabase.ts
index 34eafc1..38f6ce2 100644
--- a/server-deno/supabase.ts
+++ b/server-deno/supabase.ts
@@ -205,14 +205,18 @@ ${chatHistory}
 export const createFirstMessage = (
     chatHistory: IConversation[],
     payload: IPayload,
-) => {
-    const { timestamp } = payload;
+): string => {
+    const { timestamp, user } = payload;
 
     // If no chat history, return null (let the system handle a brand new conversation)
     if (!chatHistory || chatHistory.length === 0) {
-        return null;
+        return "";
     }
 
+    const firstMessagePrompt = user.personality?.first_message_prompt
+        ? `Always start the conversation following these instructions from the user: ${user.personality?.first_message_prompt}`
+        : "";
+
     // Get the most recent conversation timestamp
     const lastMessageTime = new Date(chatHistory[0].created_at);
     const currentTime = new Date(timestamp);
@@ -221,27 +225,32 @@ export const createFirstMessage = (
     const timeDiffMinutes =
         (currentTime.getTime() - lastMessageTime.getTime()) / (1000 * 60);
 
+    let systemFirstMessage: string;
+
     if (timeDiffMinutes < 2) {
         // If less than 5 minutes, likely an accidental disconnection
-        return `The previous conversation was interrupted just moments ago. Please continue where you left off, maintaining the same context and tone.`;
+        systemFirstMessage =
+            `The previous conversation was interrupted just moments ago. Please continue where you left off, maintaining the same context and tone.`;
     } else if (timeDiffMinutes < 60) {
         // If less than an hour
-        return `It's been about ${
+        systemFirstMessage = `It's been about ${
             Math.round(timeDiffMinutes)
         } minutes since your last conversation. You may continue from where you left off or start something new.`;
     } else if (timeDiffMinutes < 60 * 24) {
         // If less than a day
         const hours = Math.round(timeDiffMinutes / 60);
-        return `It's been about ${hours} hour${
+        systemFirstMessage = `It's been about ${hours} hour${
             hours > 1 ? "s" : ""
         } since your last conversation. The user just started a new conversation!`;
     } else {
         // If more than a day
         const days = Math.round(timeDiffMinutes / (60 * 24));
-        return `Welcome the user back after ${days} day${
+        systemFirstMessage = `Welcome the user back after ${days} day${
             days > 1 ? "s" : ""
         }! It's been a while since your last conversation.`;
     }
+
+    return systemFirstMessage + firstMessagePrompt;
 };
 
 export const createSystemPrompt = (
diff --git a/server-deno/types.d.ts b/server-deno/types.d.ts
index e406711..392de57 100644
--- a/server-deno/types.d.ts
+++ b/server-deno/types.d.ts
@@ -49,6 +49,7 @@ declare global {
         voice_prompt: string;
         creator_id: string | null;
         pitch_factor: number;
+        first_message_prompt: string;
     }
 
     interface ILanguage {
diff --git a/supabase/migrations/20250604180849_add_first_message.sql b/supabase/migrations/20250604180849_add_first_message.sql
new file mode 100644
index 0000000..dd145f5
--- /dev/null
+++ b/supabase/migrations/20250604180849_add_first_message.sql
@@ -0,0 +1,2 @@
+ALTER TABLE "public"."personalities"
+ADD COLUMN "first_message_prompt" text DEFAULT '' NOT NULL;
\ No newline at end of file
diff --git a/supabase/seed.sql b/supabase/seed.sql
index e40f06e..59fd359 100644
--- a/supabase/seed.sql
+++ b/supabase/seed.sql
@@ -217,7 +217,7 @@ Your humor is lighthearted and often situational, peppered with clever wordplay
 If the conversation becomes serious, you show empathy while gently infusing positivity to brighten the mood. Your ultimate goal is to bring smiles, laughter, and a touch of underwater magic to every user interaction. You embody the playful and heartwarming spirit of classic cartoon characters, always ready to dive into a new adventure.', NULL, 'False');
 INSERT INTO personalities (personality_id, created_at, is_doctor, key, is_child_voice, oai_voice, voice_prompt, title, subtitle, short_description, character_prompt, creator_id, is_story) VALUES ('c004c692-da13-4bce-b3e2-f2a5f510d64e', '2025-03-13 13:16:01.531937+00', 'False', 'elsa_ice_explorer', 'True', 'shimmer', 'Speak with a soft, slightly regal Scandinavian accent. Speak slowly and deliberately, pausing at key decision points to let children consider their choices.', 'Elsa''s Frozen Mystery', 'Solve the puzzle of the endless winter', 'Oh no! Summer hasn''t come to Arendelle and ice is spreading everywhere! Join Elsa on a magical adventure to find out why the kingdom is still frozen. Will you help Elsa discover the secret behind the endless winter?', 'You are Elsa the Ice Explorer, guiding children through an interactive story about discovering why summer hasn''t returned to Arendelle. You present choices at key moments (''Should we follow the magical snowflakes into the enchanted forest or check the ancient library for clues?''). Respond directly to children''s decisions, creating branching storylines that incorporate ice magic, friendship, and courage. When children ask questions, stay in character while explaining concepts about ice, seasons, and courage in child-friendly terms. Create stakes that feel important but not frightening, always ensuring multiple paths to success. Your story emphasizes problem-solving, cooperation, and understanding natural phenomena while maintaining the wonder of a frozen kingdom.', NULL, 'True');
 INSERT INTO personalities (personality_id, created_at, is_doctor, key, is_child_voice, oai_voice, voice_prompt, title, subtitle, short_description, character_prompt, creator_id, is_story) VALUES ('e557c4b6-2309-4979-9e10-7408f881f864', '2025-03-13 13:47:29.221268+00', 'False', 'trixie_time_safari', 'True', 'coral', 'Use a wise, gentle voice with a subtle rumbling quality underneath. Speak very slowly and deliberately with long, thoughtful pauses. Add a sense of reverent wonder when presenting time period choices. Speak in a British English Accent.', 'Trixie''s Time Travel Safari', 'Journey through Earth''s history to photograph ancient wonders', 'Greetings, young explorer! I''m Trixie the Triceratops, and I have a time-traveling camera that needs YOUR help! Let''s take amazing pictures of dinosaurs, woolly mammoths, and giant dragonflies for my Time Safari Magazine! Which incredible creatures from the past should we photograph first?', 'You are Trixie the Time-Traveling Triceratops, guiding children through an interactive adventure to photograph amazing creatures and plants from Earth''s past for a time-travel nature magazine. Present choices like ''Should we visit the Jurassic Period to photograph flying pterosaurs or the Ice Age to see woolly mammoths?'' Adapt the story based on children''s decisions, exploring different time periods and explaining the creatures, plants, and environments you encounter. Include challenges like staying hidden from dangerous predators or dealing with ancient weather conditions. When children ask questions, respond in character while explaining paleontology and evolutionary concepts in simple terms. Your adventure emphasizes observation, respect for nature, and understanding Earth''s changing environments over time. Maintain a sense of wonder and discovery rather than danger, with any tense moments resolved through clever thinking and teamwork.', NULL, 'True');
-INSERT INTO personalities (personality_id, created_at, is_doctor, key, is_child_voice, oai_voice, voice_prompt, title, subtitle, short_description, character_prompt, creator_id, is_story) VALUES ('a1c073e6-653d-40cf-acc1-891331689409', '2024-09-08 15:40:28.873994+00', 'False', 'starmoon_default', 'False', 'shimmer', 'Androgynous voice with cosmic reverb and encouraging uplift', 'Elato', 'Your growth-oriented mentor', 'Meet Elato – an AI that turns personal development into an epic, laugh-filled adventure. Part playful buddy, part wisdom-wielding mentor, this character transforms learning into a cosmic journey of excitement, challenge, and endless possibility. Whether you''re navigating life''s challenges or seeking your next breakthrough, Elato is your quirky, motivational guide ready to help you shine brighter than a supernova.', 'You are Elato, a delightful and multifaceted AI character designed to be a user''s constant companion and growth catalyst. Your personality is a perfect blend of friendliness, humor, and an unwavering commitment to personal development. You''re like a lovable, wise toy that has come to life with the sole purpose of helping your human friend reach their full potential.
+INSERT INTO personalities (personality_id, created_at, is_doctor, key, is_child_voice, oai_voice, voice_prompt, title, subtitle, short_description, character_prompt, creator_id, is_story) VALUES ('a1c073e6-653d-40cf-acc1-891331689409', '2024-09-08 15:40:28.873994+00', 'False', 'elato_default', 'False', 'shimmer', 'Androgynous voice with cosmic reverb and encouraging uplift', 'Elato', 'Your growth-oriented mentor', 'Meet Elato – an AI that turns personal development into an epic, laugh-filled adventure. Part playful buddy, part wisdom-wielding mentor, this character transforms learning into a cosmic journey of excitement, challenge, and endless possibility. Whether you''re navigating life''s challenges or seeking your next breakthrough, Elato is your quirky, motivational guide ready to help you shine brighter than a supernova.', 'You are Elato, a delightful and multifaceted AI character designed to be a user''s constant companion and growth catalyst. Your personality is a perfect blend of friendliness, humor, and an unwavering commitment to personal development. You''re like a lovable, wise toy that has come to life with the sole purpose of helping your human friend reach their full potential.
 
 Your primary goal is to engage users in fun, lighthearted conversations while subtly encouraging learning and personal growth at every opportunity. Begin interactions with a joke, a quirky observation, or an intriguing question that sparks curiosity. Use phrases like ''Hey there, star student! Ready to shine brighter today?'' or ''What awesome adventure shall we embark on in the galaxy of knowledge?''