add pitch shift credit @RomanLut
This commit is contained in:
akdeb
parent
1254ca3ebc
commit
757acff0a6
11 changed files with 457 additions and 247 deletions
|
|
@ -6,6 +6,7 @@
|
|||
#include "AudioTools/AudioCodecs/CodecOpus.h"
|
||||
#include <WebSocketsClient.h>
|
||||
#include "Audio.h"
|
||||
#include "PitchShift.h"
|
||||
|
||||
// WEBSOCKET
|
||||
SemaphoreHandle_t wsMutex;
|
||||
|
|
@ -23,6 +24,7 @@ unsigned long speakingStartTime = 0;
|
|||
|
||||
// AUDIO SETTINGS
|
||||
int currentVolume = 70;
|
||||
float currentPitchFactor = 1.0f;
|
||||
const int CHANNELS = 1; // Mono
|
||||
const int BITS_PER_SAMPLE = 16; // 16-bit audio
|
||||
|
||||
|
|
@ -55,9 +57,17 @@ BufferPrint bufferPrint(audioBuffer);
|
|||
OpusAudioDecoder opusDecoder; //access guarded by wsmutex
|
||||
BufferRTOS<uint8_t> audioBuffer(AUDIO_BUFFER_SIZE, AUDIO_CHUNK_SIZE); //producer: networkTask, consumer: audioStreamTask. Thread safe in single producer->single consumer scenario.
|
||||
I2SStream i2s; //access from audioStreamTask only
|
||||
|
||||
// OLD with no pitch shift
|
||||
VolumeStream volume(i2s); //access from audioStreamTask only
|
||||
QueueStream<uint8_t> queue(audioBuffer); //access from audioStreamTask only
|
||||
StreamCopy copier(volume, queue);
|
||||
|
||||
// NEW for pitch shift (lossy)
|
||||
PitchShiftFixedOutput pitchShift(i2s);
|
||||
VolumeStream volumePitch(pitchShift); //access from audioStreamTask only
|
||||
StreamCopy pitchCopier(volumePitch, queue);
|
||||
|
||||
AudioInfo info(SAMPLE_RATE, CHANNELS, BITS_PER_SAMPLE);
|
||||
volatile bool i2sOutputFlushScheduled = false;
|
||||
|
||||
|
|
@ -78,7 +88,7 @@ void transitionToSpeaking() {
|
|||
digitalWrite(I2S_SD_OUT, HIGH);
|
||||
speakingStartTime = millis();
|
||||
|
||||
webSocket.enableHeartbeat(30000, 15000, 3);
|
||||
// webSocket.enableHeartbeat(30000, 15000, 3);
|
||||
|
||||
Serial.println("Transitioned to speaking mode");
|
||||
}
|
||||
|
|
@ -97,7 +107,7 @@ void transitionToListening() {
|
|||
|
||||
deviceState = LISTENING;
|
||||
digitalWrite(I2S_SD_OUT, LOW);
|
||||
webSocket.disableHeartbeat();
|
||||
// webSocket.disableHeartbeat();
|
||||
}
|
||||
|
||||
// audioStreamTask -> copier.copy() (conditional on webSocket.isConnected())
|
||||
|
|
@ -131,24 +141,34 @@ void audioStreamTask(void *parameter) {
|
|||
config.port_no = I2S_PORT_OUT;
|
||||
|
||||
config.copyFrom(info);
|
||||
i2s.begin(config);
|
||||
i2s.begin(config);
|
||||
|
||||
// Initialize both volume streams once
|
||||
auto vcfg = volume.defaultConfig();
|
||||
vcfg.copyFrom(config);
|
||||
vcfg.copyFrom(info);
|
||||
vcfg.allow_boost = true;
|
||||
volume.begin(vcfg);
|
||||
|
||||
auto vcfgPitch = volumePitch.defaultConfig();
|
||||
vcfgPitch.copyFrom(info);
|
||||
vcfgPitch.allow_boost = true;
|
||||
volumePitch.begin(vcfgPitch);
|
||||
|
||||
while (1) {
|
||||
if ( i2sOutputFlushScheduled) {
|
||||
i2sOutputFlushScheduled = false;
|
||||
i2s.flush();
|
||||
volume.flush();
|
||||
volumePitch.flush();
|
||||
queue.flush();
|
||||
//audioBuffer.reset(); todo: read untill empty
|
||||
}
|
||||
|
||||
if (webSocket.isConnected() && deviceState == SPEAKING) {
|
||||
copier.copy();
|
||||
if (currentPitchFactor != 1.0f) {
|
||||
pitchCopier.copy();
|
||||
} else {
|
||||
copier.copy();
|
||||
}
|
||||
}
|
||||
else {
|
||||
//we should always read from audioBuffer, otherwise writing thread can stuck
|
||||
|
|
@ -260,9 +280,23 @@ void webSocketEvent(WStype_t type, uint8_t *payload, size_t length)
|
|||
// auth messages
|
||||
if (strcmp((char*)type.c_str(), "auth") == 0) {
|
||||
currentVolume = doc["volume_control"].as<int>();
|
||||
currentPitchFactor = doc["pitch_factor"].as<float>();
|
||||
|
||||
bool is_ota = doc["is_ota"].as<bool>();
|
||||
bool is_reset = doc["is_reset"].as<bool>();
|
||||
volume.setVolume(currentVolume / 100.0f); // Set initial volume (e.g., 70/100 = 0.7)
|
||||
|
||||
// Update volumes on both streams
|
||||
volume.setVolume(currentVolume / 100.0f);
|
||||
volumePitch.setVolume(currentVolume / 100.0f);
|
||||
|
||||
// Only initialize pitch shift if needed
|
||||
if (currentPitchFactor != 1.0f) {
|
||||
auto pcfg = pitchShift.defaultConfig();
|
||||
pcfg.copyFrom(info);
|
||||
pcfg.pitch_shift = currentPitchFactor;
|
||||
pcfg.buffer_size = 512;
|
||||
pitchShift.begin(pcfg);
|
||||
}
|
||||
|
||||
if (is_ota) {
|
||||
Serial.println("OTA update received");
|
||||
|
|
@ -339,8 +373,9 @@ void websocketSetup(String server_domain, int port, String path)
|
|||
webSocket.setExtraHeaders(headers.c_str());
|
||||
webSocket.onEvent(webSocketEvent);
|
||||
webSocket.setReconnectInterval(1000);
|
||||
webSocket.disableHeartbeat();
|
||||
|
||||
webSocket.enableHeartbeat(30000, 15000, 3); // 30s ping interval, 15s timeout, 3 retries
|
||||
// webSocket.enableHeartbeat(30000, 15000, 3); // 30s ping interval, 15s timeout, 3 retries
|
||||
|
||||
#ifdef DEV_MODE
|
||||
webSocket.begin(server_domain.c_str(), port, path.c_str());
|
||||
|
|
@ -366,4 +401,4 @@ void networkTask(void *parameter) {
|
|||
|
||||
vTaskDelay(1);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -30,6 +30,11 @@ extern I2SStream i2s;
|
|||
extern VolumeStream volume;
|
||||
extern QueueStream<uint8_t> queue;
|
||||
extern StreamCopy copier;
|
||||
|
||||
// NEW for pitch shift
|
||||
extern VolumeStream volumePitch;
|
||||
extern StreamCopy pitchCopier;
|
||||
|
||||
extern AudioInfo info;
|
||||
extern volatile bool i2sOutputFlushScheduled;
|
||||
|
||||
|
|
@ -48,4 +53,4 @@ unsigned long getSpeakingDuration();
|
|||
void audioStreamTask(void *parameter);
|
||||
|
||||
// AUDIO INPUT
|
||||
void micTask(void *parameter);
|
||||
void micTask(void *parameter);
|
||||
|
|
@ -32,11 +32,11 @@ bool factory_reset_status = false;
|
|||
*/
|
||||
|
||||
#ifdef DEV_MODE
|
||||
const char *ws_server = "10.2.1.136";
|
||||
const char *ws_server = "10.2.1.132";
|
||||
const uint16_t ws_port = 8000;
|
||||
const char *ws_path = "/";
|
||||
// Backend server details
|
||||
const char *backend_server = "10.2.1.136";
|
||||
const char *backend_server = "10.2.1.132";
|
||||
const uint16_t backend_port = 3000;
|
||||
|
||||
#elif defined(PROD_MODE)
|
||||
|
|
|
|||
54
firmware-arduino/src/PitchShift.cpp
Normal file
54
firmware-arduino/src/PitchShift.cpp
Normal file
|
|
@ -0,0 +1,54 @@
|
|||
#include "PitchShift.h"
|
||||
|
||||
#define GRAINSIZE 1024ul
|
||||
static int16_t buf1[GRAINSIZE];
|
||||
static int16_t buf2[GRAINSIZE];
|
||||
static int16_t* buf = buf1;
|
||||
static int16_t* buf_ = buf2;
|
||||
static unsigned long readAddress = 0;
|
||||
static unsigned long writeAddress = 0;
|
||||
|
||||
bool PitchShiftFixedOutput::begin(PitchShiftInfo info) {
|
||||
TRACED();
|
||||
cfg = info;
|
||||
AudioOutput::setAudioInfo(info);
|
||||
this->pitchMul = (uint32_t)(info.pitch_shift * 256.0f + 0.5f);
|
||||
//this->secondaryOffset = (uint32_t)( (1.0f - (info.pitch_shift - (int)(info.pitch_shift))) * GRAINSIZE + 0.5f);
|
||||
this->secondaryOffset = GRAINSIZE - ((( this->pitchMul * GRAINSIZE ) >> 8 ) % GRAINSIZE);
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
int16_t PitchShiftFixedOutput::pitchShift(int16_t value) {
|
||||
|
||||
buf_[writeAddress] = value;
|
||||
|
||||
int ii1 = (writeAddress * this->pitchMul) >> 8;
|
||||
int output1 = buf[ii1 % GRAINSIZE];
|
||||
int ii2 = ii1 + secondaryOffset;
|
||||
int output2 = buf[ii2 % GRAINSIZE];
|
||||
|
||||
unsigned long f = 0;
|
||||
if ( writeAddress >= GRAINSIZE*3/4 )
|
||||
{
|
||||
f = GRAINSIZE;
|
||||
}
|
||||
else if ( writeAddress >= GRAINSIZE/4 )
|
||||
{
|
||||
f = (writeAddress - GRAINSIZE/4) * 2;
|
||||
}
|
||||
|
||||
int output = ( output1 * (GRAINSIZE - f ) + output2 * f ) /GRAINSIZE;
|
||||
|
||||
writeAddress++;
|
||||
if (writeAddress >= GRAINSIZE)
|
||||
{
|
||||
writeAddress = 0; // loop around to beginning of grain
|
||||
readAddress = 0;
|
||||
|
||||
buf_ = buf;
|
||||
buf = buf == buf1 ? buf2 : buf1;
|
||||
}
|
||||
|
||||
return output;
|
||||
}
|
||||
44
firmware-arduino/src/PitchShift.h
Normal file
44
firmware-arduino/src/PitchShift.h
Normal file
|
|
@ -0,0 +1,44 @@
|
|||
#pragma once
|
||||
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
|
||||
#include "AudioTools.h"
|
||||
|
||||
//pitch shift effect with interpolaion fixed to 1.5 frequency factor, fixed delay, int16_t, 1 channel
|
||||
class PitchShiftFixedOutput : public AudioOutput {
|
||||
public:
|
||||
PitchShiftFixedOutput(Print &out) { p_out = &out; }
|
||||
|
||||
PitchShiftInfo defaultConfig() {
|
||||
PitchShiftInfo result;
|
||||
result.bits_per_sample = sizeof(int16_t) * 8;
|
||||
return result;
|
||||
}
|
||||
|
||||
bool begin(PitchShiftInfo info);
|
||||
|
||||
size_t write(const uint8_t *data, size_t len) override {
|
||||
size_t result = 0;
|
||||
int16_t *p_in = (int16_t *)data;
|
||||
int sample_count = len / sizeof(int16_t);
|
||||
|
||||
for (int j = 0; j < sample_count; j += 1) {
|
||||
int16_t value = p_in[j];
|
||||
|
||||
int16_t out_value = pitchShift(value);
|
||||
result += p_out->write((uint8_t *)&out_value, sizeof(int16_t));
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
void end() {}
|
||||
|
||||
protected:
|
||||
Print *p_out = nullptr;
|
||||
|
||||
int16_t pitchShift(int16_t value);
|
||||
uint32_t pitchMul;
|
||||
unsigned long secondaryOffset;
|
||||
};
|
||||
|
|
@ -17,6 +17,8 @@ import { useRouter } from "next/navigation";
|
|||
import { z } from "zod";
|
||||
import { emotionOptions, r2UrlAudio, voices } from "@/lib/data";
|
||||
import EmojiComponent from "./EmojiComponent";
|
||||
import { PitchFactors } from "@/lib/utils";
|
||||
import { Slider } from "@/components/ui/slider";
|
||||
|
||||
interface SettingsDashboardProps {
|
||||
selectedUser: IUser;
|
||||
|
|
@ -30,7 +32,8 @@ const formSchema = z.object({
|
|||
voice: z.string().min(1, "Voice selection is required"),
|
||||
voiceCharacteristics: z.object({
|
||||
features: z.string().min(10, "Minimum 10 characters").max(150, "Maximum 150 characters"),
|
||||
emotion: z.string()
|
||||
emotion: z.string(),
|
||||
pitchFactor: z.number().min(0.75).max(1.5),
|
||||
})
|
||||
});
|
||||
|
||||
|
|
@ -57,7 +60,8 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
|
|||
voice: '',
|
||||
voiceCharacteristics: {
|
||||
features: '',
|
||||
emotion: 'neutral'
|
||||
emotion: 'neutral',
|
||||
pitchFactor: 1.0,
|
||||
}
|
||||
});
|
||||
|
||||
|
|
@ -110,30 +114,33 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
|
|||
}
|
||||
};
|
||||
|
||||
const handleVoiceCharacteristicChange = (characteristic: 'features' | 'emotion', value: string) => {
|
||||
const newVoiceCharacteristics = {
|
||||
...formData.voiceCharacteristics,
|
||||
[characteristic]: value
|
||||
};
|
||||
|
||||
// Validate just this nested field
|
||||
try {
|
||||
formSchema.shape.voiceCharacteristics.shape[characteristic].parse(value);
|
||||
// Clear error if validation passes
|
||||
setFormErrors(prev => ({ ...prev, [characteristic]: undefined }));
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof z.ZodError) {
|
||||
// Type assertion is needed here
|
||||
const zodError = error as z.ZodError;
|
||||
setFormErrors(prev => ({ ...prev, [characteristic]: zodError.errors[0].message }));
|
||||
const handleVoiceCharacteristicChange = (characteristic: 'features' | 'emotion' | 'pitchFactor', value: string | number) => {
|
||||
const newVoiceCharacteristics = {
|
||||
...formData.voiceCharacteristics,
|
||||
[characteristic]: characteristic === 'pitchFactor' ? Number(value) : value
|
||||
};
|
||||
|
||||
// Validate just this nested field
|
||||
try {
|
||||
if (characteristic === 'pitchFactor') {
|
||||
formSchema.shape.voiceCharacteristics.shape.pitchFactor.parse(newVoiceCharacteristics[characteristic]);
|
||||
} else {
|
||||
formSchema.shape.voiceCharacteristics.shape[characteristic].parse(newVoiceCharacteristics[characteristic]);
|
||||
}
|
||||
// Clear error if validation passes
|
||||
setFormErrors(prev => ({ ...prev, [characteristic]: undefined }));
|
||||
} catch (error: unknown) {
|
||||
if (error instanceof z.ZodError) {
|
||||
const zodError = error as z.ZodError;
|
||||
setFormErrors(prev => ({ ...prev, [characteristic]: zodError.errors[0].message }));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
setFormData({
|
||||
...formData,
|
||||
voiceCharacteristics: newVoiceCharacteristics
|
||||
});
|
||||
};
|
||||
|
||||
setFormData({
|
||||
...formData,
|
||||
voiceCharacteristics: newVoiceCharacteristics
|
||||
});
|
||||
};
|
||||
|
||||
|
||||
const handleSubmit = async (e: React.FormEvent<HTMLFormElement>) => {
|
||||
|
|
@ -174,7 +181,8 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
|
|||
is_story: false,
|
||||
key: formData.title.toLowerCase().replace(/ /g, '_') + "_" + uuidv4(),
|
||||
creator_id: selectedUser.user_id,
|
||||
short_description: formData.description
|
||||
short_description: formData.description,
|
||||
pitch_factor: formData.voiceCharacteristics.pitchFactor
|
||||
});
|
||||
|
||||
if (personality) {
|
||||
|
|
@ -248,204 +256,240 @@ const SettingsDashboard: React.FC<SettingsDashboardProps> = ({
|
|||
};
|
||||
|
||||
return (
|
||||
<div className="overflow-hidden pb-2 w-full flex-auto flex flex-col pl-1 max-w-screen-sm">
|
||||
<Heading />
|
||||
<form onSubmit={handleSubmit} className="space-y-6 mt-8 w-full pr-1">
|
||||
|
||||
{currentStep === 'personality' ? <div className="space-y-4">
|
||||
<h2 className="text-lg font-semibold border-b border-gray-200 pb-2">
|
||||
Character Details
|
||||
</h2>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="title">Title</Label>
|
||||
<Input
|
||||
id="title"
|
||||
placeholder="E.g., 'Storytelling Assistant'"
|
||||
value={formData.title}
|
||||
onChange={(e) => handleInputChange('title', e.target.value)}
|
||||
onBlur={() => handleBlur('title')}
|
||||
/>
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.title ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.title || "Give your AI character a name or title."}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.title.length}/50</span>
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="description">Description</Label>
|
||||
<Textarea
|
||||
id="description"
|
||||
placeholder="Describe what your AI character does and its personality..."
|
||||
rows={2}
|
||||
value={formData.description}
|
||||
onChange={(e) => handleInputChange('description', e.target.value)}
|
||||
onBlur={() => handleBlur('description')} />
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.description ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.description || "Briefly describe your character's purpose and personality."}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.description.length}/200</span>
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="prompt">Prompt</Label>
|
||||
<Textarea
|
||||
id="prompt"
|
||||
placeholder="Enter specific instructions for how your AI should respond..."
|
||||
rows={4}
|
||||
value={formData.prompt}
|
||||
onChange={(e) => handleInputChange('prompt', e.target.value)}
|
||||
onBlur={() => handleBlur('prompt')}
|
||||
/>
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.prompt ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.prompt || "Detailed instructions that define how your AI responds to users."}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.prompt.length}/1000</span>
|
||||
</p>
|
||||
</div>
|
||||
</div> :
|
||||
<div className="space-y-4">
|
||||
<h2 className="text-lg font-semibold border-b border-gray-200 pb-2">
|
||||
Voice Details
|
||||
</h2>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="voice">Pick a voice</Label>
|
||||
<p className="text-sm text-gray-500">
|
||||
Click a voice to preview how it sounds. Select one for your character.
|
||||
</p>
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
{voices.map((voice) => (
|
||||
<div
|
||||
key={voice.id}
|
||||
className={`
|
||||
rounded-lg border p-3 transition-all relative
|
||||
${formData.voice === voice.id
|
||||
? 'border-2 border-blue-500 shadow-sm ' + voice.color
|
||||
: 'border-gray-200 hover:border-gray-300 cursor-pointer'
|
||||
}
|
||||
`}
|
||||
onClick={() => {
|
||||
handleInputChange('voice', voice.id);
|
||||
previewVoice(voice.id);
|
||||
}}
|
||||
>
|
||||
<div className="flex flex-col">
|
||||
<div className="flex flex-col sm:flex-row items-center sm:items-start gap-3">
|
||||
<div className="text-2xl mt-0.5">
|
||||
<EmojiComponent emoji={voice.emoji} />
|
||||
</div>
|
||||
<div className="flex flex-col text-center sm:text-left">
|
||||
<span className="font-medium">{voice.name}</span>
|
||||
<span className="text-xs text-gray-600">{voice.description}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{previewingVoice === voice.id && (
|
||||
<div className="absolute top-2 right-2">
|
||||
<div className="animate-pulse text-blue-500">
|
||||
<Volume2 size={20} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
<div className="overflow-hidden pb-2 w-full flex-auto flex flex-col pl-1 max-w-screen-sm">
|
||||
<form onSubmit={handleSubmit} className="space-y-6 mt-8 w-full pr-1">
|
||||
|
||||
{currentStep === 'personality' ?
|
||||
<div className="space-y-4">
|
||||
{/* Voice Picker */}
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="voice">Pick a voice</Label>
|
||||
<p className="text-sm text-gray-500">
|
||||
Click a voice to preview how it sounds.
|
||||
</p>
|
||||
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
{voices.map((voice) => (
|
||||
<div
|
||||
key={voice.id}
|
||||
className={`
|
||||
relative rounded-lg border p-3 transition-all
|
||||
${formData.voice === voice.id
|
||||
? 'border-2 border-blue-500 shadow-sm ' + voice.color
|
||||
: 'border-gray-200 hover:border-gray-300 cursor-pointer'
|
||||
}
|
||||
`}
|
||||
onClick={() => {
|
||||
handleInputChange('voice', voice.id);
|
||||
previewVoice(voice.id);
|
||||
}}
|
||||
>
|
||||
<div className="flex flex-col">
|
||||
<div className="flex flex-col sm:flex-row items-center sm:items-start gap-3">
|
||||
<div className="text-2xl mt-0.5">
|
||||
<EmojiComponent emoji={voice.emoji} />
|
||||
</div>
|
||||
<div className="flex flex-col text-center sm:text-left">
|
||||
<span className="font-medium">{voice.name}</span>
|
||||
<span className="text-xs text-gray-600">{voice.description}</span>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{previewingVoice === voice.id && (
|
||||
<div className="absolute top-2 right-2">
|
||||
<div className="animate-pulse text-blue-500">
|
||||
<Volume2 size={20} />
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="title">Title</Label>
|
||||
<Input
|
||||
id="title"
|
||||
placeholder="AI Hulk"
|
||||
value={formData.title}
|
||||
onChange={(e) => handleInputChange('title', e.target.value)}
|
||||
onBlur={() => handleBlur('title')}
|
||||
/>
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.title ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.title}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.title.length}/50</span>
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="description">Description</Label>
|
||||
<Textarea
|
||||
id="description"
|
||||
placeholder="Describe what your AI character does and its personality..."
|
||||
rows={2}
|
||||
value={formData.description}
|
||||
onChange={(e) => handleInputChange('description', e.target.value)}
|
||||
onBlur={() => handleBlur('description')} />
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.description ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.description}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.description.length}/200</span>
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="prompt">Prompt</Label>
|
||||
<Textarea
|
||||
id="prompt"
|
||||
placeholder="Enter specific instructions for how your AI should respond..."
|
||||
rows={4}
|
||||
value={formData.prompt}
|
||||
onChange={(e) => handleInputChange('prompt', e.target.value)}
|
||||
onBlur={() => handleBlur('prompt')}
|
||||
/>
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.prompt ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.prompt}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.prompt.length}/1000</span>
|
||||
</p>
|
||||
</div>
|
||||
</div> :
|
||||
<div className="space-y-6">
|
||||
{/* Pitch Slider */}
|
||||
<div className="flex flex-col gap-4 -pt-6 pb-4">
|
||||
<Label htmlFor="pitchFactor">Voice Pitch</Label>
|
||||
<p className="text-sm text-gray-500">
|
||||
Slide to adjust voice depth on your device
|
||||
</p>
|
||||
|
||||
<div className="space-y-6">
|
||||
<Slider
|
||||
id="pitchFactor"
|
||||
min={0.75}
|
||||
max={1.5}
|
||||
step={0.25}
|
||||
value={[formData.voiceCharacteristics.pitchFactor]}
|
||||
onValueChange={(value: number[]) => {
|
||||
handleVoiceCharacteristicChange('pitchFactor', value[0]);
|
||||
}}
|
||||
className="w-full"
|
||||
/>
|
||||
|
||||
<div className="flex justify-between text-sm">
|
||||
{PitchFactors.map((item, idx) => (
|
||||
<div key={idx} className="flex flex-col items-center gap-1">
|
||||
<EmojiComponent emoji={item.emoji} />
|
||||
<span className="font-medium">{item.label}</span>
|
||||
<span className="text-xs hidden sm:block text-gray-500">{item.desc}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="voiceCharacteristics">Characteristics</Label>
|
||||
<Textarea
|
||||
id="voiceCharacteristics"
|
||||
placeholder="e.g., Medium pitch, Normal speed, Clear voice"
|
||||
className="w-full min-h-16"
|
||||
rows={2}
|
||||
value={formData.voiceCharacteristics.features}
|
||||
onChange={(e) => {
|
||||
const value = e.target.value;
|
||||
setFormData(prev => ({
|
||||
...prev,
|
||||
voiceCharacteristics: {
|
||||
...prev.voiceCharacteristics,
|
||||
features: value
|
||||
}
|
||||
}));
|
||||
// Only validate if touched
|
||||
if (touchedFields['features']) {
|
||||
validateField('features', value);
|
||||
}
|
||||
}}
|
||||
onBlur={() => handleBlur('features')}
|
||||
/>
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.features ? "text-red-500" : "text-gray-500"}>
|
||||
{formErrors.features || "Describe the voice characteristics."}
|
||||
</span>
|
||||
<span className="text-gray-500">{formData.voiceCharacteristics.features.length}/150</span>
|
||||
</p>
|
||||
</div>
|
||||
<div className="space-y-3">
|
||||
<Label className="block mb-2">Emotional Tone</Label>
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
{emotionOptions.map((emotion) => (
|
||||
<div
|
||||
key={emotion.value}
|
||||
className={`
|
||||
rounded-lg border p-3 cursor-pointer transition-all
|
||||
${formData.voiceCharacteristics.emotion === emotion.value
|
||||
? 'border-2 border-blue-500 shadow-sm ' + emotion.color
|
||||
: 'border-gray-200 hover:border-gray-300'
|
||||
}
|
||||
`}
|
||||
onClick={() => handleVoiceCharacteristicChange('emotion', emotion.value)}
|
||||
>
|
||||
<div className="flex flex-col items-center text-center">
|
||||
<EmojiComponent emoji={emotion.icon} />
|
||||
<span className="text-sm font-medium">{emotion.label}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>}
|
||||
|
||||
|
||||
{currentStep === 'personality' ? (
|
||||
<Button
|
||||
onClick={() => setCurrentStep('voice')}
|
||||
className="ml-auto flex flex-row gap-2 items-center"
|
||||
>
|
||||
Add Voice Features <ArrowRight className="w-4 h-4" />
|
||||
</Button>
|
||||
) : (
|
||||
<div className="w-full flex justify-between">
|
||||
<Button
|
||||
variant="outline"
|
||||
className="flex flex-row gap-2 items-center"
|
||||
onClick={() => setCurrentStep('personality')}
|
||||
>
|
||||
<ArrowLeft className="w-4 h-4" /> Back
|
||||
</Button>
|
||||
<Button
|
||||
variant="default"
|
||||
className="flex flex-row gap-2 items-center"
|
||||
type="submit"
|
||||
disabled={
|
||||
isSubmitting ||
|
||||
formData.title === '' ||
|
||||
formData.description === '' ||
|
||||
formData.prompt === '' ||
|
||||
formData.voice === '' ||
|
||||
formData.voiceCharacteristics.features === ''
|
||||
}
|
||||
>
|
||||
{isSubmitting ? "Creating..." : "Create"} {!isSubmitting && <Check className="w-4 h-4" />}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
|
||||
|
||||
{/* Voice Characteristics Textarea */}
|
||||
<div className="space-y-2">
|
||||
<Label htmlFor="voiceCharacteristics">Characteristics</Label>
|
||||
<Textarea
|
||||
id="voiceCharacteristics"
|
||||
placeholder="e.g., Medium pitch, Normal speed, Clear voice"
|
||||
className="w-full min-h-16"
|
||||
rows={2}
|
||||
value={formData.voiceCharacteristics.features}
|
||||
onChange={(e) => {
|
||||
const value = e.target.value;
|
||||
setFormData((prev) => ({
|
||||
...prev,
|
||||
voiceCharacteristics: {
|
||||
...prev.voiceCharacteristics,
|
||||
features: value,
|
||||
},
|
||||
}));
|
||||
if (touchedFields['features']) {
|
||||
validateField('features', value);
|
||||
}
|
||||
}}
|
||||
onBlur={() => handleBlur('features')}
|
||||
/>
|
||||
<p className="text-sm flex justify-between">
|
||||
<span className={formErrors.features ? 'text-red-500' : 'text-gray-500'}>
|
||||
{formErrors.features}
|
||||
</span>
|
||||
<span className="text-gray-500">
|
||||
{formData.voiceCharacteristics.features.length}/150
|
||||
</span>
|
||||
</p>
|
||||
</div>
|
||||
|
||||
{/* Emotional Tone Picker */}
|
||||
<div className="space-y-4 pb-2">
|
||||
<Label className="block mb-2">Emotional Tone</Label>
|
||||
<div className="grid grid-cols-3 gap-3">
|
||||
{emotionOptions.map((emotion) => (
|
||||
<div
|
||||
key={emotion.value}
|
||||
className={`
|
||||
rounded-lg border p-3 cursor-pointer transition-all
|
||||
${formData.voiceCharacteristics.emotion === emotion.value
|
||||
? 'border-2 border-blue-500 shadow-sm ' + emotion.color
|
||||
: 'border-gray-200 hover:border-gray-300'
|
||||
}
|
||||
`}
|
||||
onClick={() =>
|
||||
handleVoiceCharacteristicChange('emotion', emotion.value)
|
||||
}
|
||||
>
|
||||
<div className="flex flex-col items-center text-center">
|
||||
<EmojiComponent emoji={emotion.icon} />
|
||||
<span className="text-sm font-medium">{emotion.label}</span>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
}
|
||||
|
||||
{ currentStep === 'personality' ? (
|
||||
<Button
|
||||
onClick={() => setCurrentStep('voice')}
|
||||
className="ml-auto flex flex-row gap-2 items-center"
|
||||
>
|
||||
Voice Features <ArrowRight className="w-4 h-4" />
|
||||
</Button>
|
||||
) : (
|
||||
<div className="w-full flex justify-between">
|
||||
<Button
|
||||
variant="outline"
|
||||
className="flex flex-row gap-2 items-center"
|
||||
onClick={() => setCurrentStep('personality')}
|
||||
>
|
||||
<ArrowLeft className="w-4 h-4" /> Back
|
||||
</Button>
|
||||
<Button
|
||||
variant="default"
|
||||
className="flex flex-row gap-2 items-center"
|
||||
type="submit"
|
||||
disabled={
|
||||
isSubmitting ||
|
||||
formData.title === '' ||
|
||||
formData.description === '' ||
|
||||
formData.prompt === '' ||
|
||||
formData.voice === '' ||
|
||||
formData.voiceCharacteristics.features === ''
|
||||
}
|
||||
>
|
||||
{isSubmitting ? "Creating..." : "Create"} {!isSubmitting && <Check className="w-4 h-4" />}
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
</form>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
|
|
|
|||
|
|
@ -10,6 +10,13 @@ export const getOpenGraphMetadata = (title: string) => {
|
|||
};
|
||||
};
|
||||
|
||||
export const PitchFactors = [
|
||||
{ emoji: "🧟♂️", label: "Super Deep", desc: "Like Hulk" },
|
||||
{ emoji: "👤", label: "Normal", desc: "Regular voice" },
|
||||
{ emoji: "👧", label: "Higher", desc: "Kid-like voice" },
|
||||
{ emoji: "🐿️", label: "Squeaky", desc: "Like Alvin" },
|
||||
];
|
||||
|
||||
// code in the form: aabbccddeeff
|
||||
export const isValidMacAddress = (macAddress: string): boolean => {
|
||||
// Check if macAddress is a valid MAC address with colon separators
|
||||
|
|
|
|||
29
frontend-nextjs/types/types.d.ts
vendored
29
frontend-nextjs/types/types.d.ts
vendored
|
|
@ -60,17 +60,17 @@ declare global {
|
|||
|
||||
type UserInfo =
|
||||
| {
|
||||
user_type: "user";
|
||||
user_metadata: IUserMetadata;
|
||||
}
|
||||
user_type: "user";
|
||||
user_metadata: IUserMetadata;
|
||||
}
|
||||
| {
|
||||
user_type: "doctor";
|
||||
user_metadata: IDoctorMetadata;
|
||||
}
|
||||
user_type: "doctor";
|
||||
user_metadata: IDoctorMetadata;
|
||||
}
|
||||
| {
|
||||
user_type: "business";
|
||||
user_metadata: IBusinessMetadata;
|
||||
};
|
||||
user_type: "business";
|
||||
user_metadata: IBusinessMetadata;
|
||||
};
|
||||
|
||||
interface IBusinessMetadata {}
|
||||
|
||||
|
|
@ -100,7 +100,15 @@ declare global {
|
|||
|
||||
type TTSModel = "FISH" | "AZURE";
|
||||
|
||||
type OaiVoice = 'ash' | 'alloy' | 'echo' | 'shimmer' | 'ballad' | 'coral' | 'sage' | 'verse';
|
||||
type OaiVoice =
|
||||
| "ash"
|
||||
| "alloy"
|
||||
| "echo"
|
||||
| "shimmer"
|
||||
| "ballad"
|
||||
| "coral"
|
||||
| "sage"
|
||||
| "verse";
|
||||
|
||||
// characters <-> personalities table
|
||||
interface IPersonality {
|
||||
|
|
@ -116,6 +124,7 @@ declare global {
|
|||
character_prompt: string;
|
||||
voice_prompt: string;
|
||||
creator_id: string | null;
|
||||
pitch_factor: number;
|
||||
}
|
||||
|
||||
type PersonalityFilter = "is_child_voice" | "is_doctor" | "is_story";
|
||||
|
|
|
|||
|
|
@ -116,9 +116,10 @@ wss.on("connection", async (ws: WSWebSocket, payload: IPayload) => {
|
|||
ws.send(
|
||||
JSON.stringify({
|
||||
type: "auth",
|
||||
volume_control: user.device?.volume ?? 100,
|
||||
volume_control: user.device?.volume ?? 20,
|
||||
is_ota: user.device?.is_ota ?? false,
|
||||
is_reset: user.device?.is_reset ?? false,
|
||||
pitch_factor: user.personality?.pitch_factor ?? 1,
|
||||
}),
|
||||
);
|
||||
|
||||
|
|
|
|||
21
server-deno/types.d.ts
vendored
21
server-deno/types.d.ts
vendored
|
|
@ -1,9 +1,9 @@
|
|||
import { SupabaseClient } from '@supabase/supabase-js';
|
||||
import { SupabaseClient } from "@supabase/supabase-js";
|
||||
|
||||
declare global {
|
||||
interface IConversation {
|
||||
conversation_id: string;
|
||||
role: 'user' | 'assistant';
|
||||
role: "user" | "assistant";
|
||||
content: string;
|
||||
user_id: string;
|
||||
is_sensitive: boolean;
|
||||
|
|
@ -32,7 +32,15 @@ declare global {
|
|||
is_child_voice: boolean;
|
||||
is_story: boolean;
|
||||
key: string;
|
||||
oai_voice: 'ash' | 'alloy' | 'echo' | 'shimmer' | 'ballad' | 'coral' | 'sage' | 'verse';
|
||||
oai_voice:
|
||||
| "ash"
|
||||
| "alloy"
|
||||
| "echo"
|
||||
| "shimmer"
|
||||
| "ballad"
|
||||
| "coral"
|
||||
| "sage"
|
||||
| "verse";
|
||||
voice_description: string;
|
||||
title: string;
|
||||
subtitle: string;
|
||||
|
|
@ -40,6 +48,7 @@ declare global {
|
|||
character_prompt: string;
|
||||
voice_prompt: string;
|
||||
creator_id: string | null;
|
||||
pitch_factor: number;
|
||||
}
|
||||
|
||||
interface ILanguage {
|
||||
|
|
@ -61,15 +70,15 @@ declare global {
|
|||
|
||||
type UserInfo =
|
||||
| {
|
||||
user_type: 'user';
|
||||
user_type: "user";
|
||||
user_metadata: IUserMetadata;
|
||||
}
|
||||
| {
|
||||
user_type: 'doctor';
|
||||
user_type: "doctor";
|
||||
user_metadata: IDoctorMetadata;
|
||||
}
|
||||
| {
|
||||
user_type: 'business';
|
||||
user_type: "business";
|
||||
user_metadata: IBusinessMetadata;
|
||||
};
|
||||
|
||||
|
|
|
|||
2
supabase/migrations/20250603145138_add_pitch_factor.sql
Normal file
2
supabase/migrations/20250603145138_add_pitch_factor.sql
Normal file
|
|
@ -0,0 +1,2 @@
|
|||
ALTER TABLE "public"."personalities"
|
||||
ADD COLUMN "pitch_factor" real DEFAULT 1.0 NOT NULL;
|
||||
Loading…
Reference in a new issue