From 757acff0a61739be747d3586a41a235e99f539e2 Mon Sep 17 00:00:00 2001 From: akdeb Date: Tue, 3 Jun 2025 16:03:42 +0100 Subject: [PATCH] add pitch shift credit @RomanLut --- firmware-arduino/src/Audio.cpp | 53 +- firmware-arduino/src/Audio.h | 7 +- firmware-arduino/src/Config.cpp | 4 +- firmware-arduino/src/PitchShift.cpp | 54 ++ firmware-arduino/src/PitchShift.h | 44 ++ .../CreateCharacter/BuildDashboard.tsx | 480 ++++++++++-------- frontend-nextjs/lib/utils.ts | 7 + frontend-nextjs/types/types.d.ts | 29 +- server-deno/main.ts | 3 +- server-deno/types.d.ts | 21 +- .../20250603145138_add_pitch_factor.sql | 2 + 11 files changed, 457 insertions(+), 247 deletions(-) create mode 100644 firmware-arduino/src/PitchShift.cpp create mode 100644 firmware-arduino/src/PitchShift.h create mode 100644 supabase/migrations/20250603145138_add_pitch_factor.sql diff --git a/firmware-arduino/src/Audio.cpp b/firmware-arduino/src/Audio.cpp index dd22de5..12d64be 100644 --- a/firmware-arduino/src/Audio.cpp +++ b/firmware-arduino/src/Audio.cpp @@ -6,6 +6,7 @@ #include "AudioTools/AudioCodecs/CodecOpus.h" #include #include "Audio.h" +#include "PitchShift.h" // WEBSOCKET SemaphoreHandle_t wsMutex; @@ -23,6 +24,7 @@ unsigned long speakingStartTime = 0; // AUDIO SETTINGS int currentVolume = 70; +float currentPitchFactor = 1.0f; const int CHANNELS = 1; // Mono const int BITS_PER_SAMPLE = 16; // 16-bit audio @@ -55,9 +57,17 @@ BufferPrint bufferPrint(audioBuffer); OpusAudioDecoder opusDecoder; //access guarded by wsmutex BufferRTOS audioBuffer(AUDIO_BUFFER_SIZE, AUDIO_CHUNK_SIZE); //producer: networkTask, consumer: audioStreamTask. Thread safe in single producer->single consumer scenario. I2SStream i2s; //access from audioStreamTask only + +// OLD with no pitch shift VolumeStream volume(i2s); //access from audioStreamTask only QueueStream queue(audioBuffer); //access from audioStreamTask only StreamCopy copier(volume, queue); + +// NEW for pitch shift (lossy) +PitchShiftFixedOutput pitchShift(i2s); +VolumeStream volumePitch(pitchShift); //access from audioStreamTask only +StreamCopy pitchCopier(volumePitch, queue); + AudioInfo info(SAMPLE_RATE, CHANNELS, BITS_PER_SAMPLE); volatile bool i2sOutputFlushScheduled = false; @@ -78,7 +88,7 @@ void transitionToSpeaking() { digitalWrite(I2S_SD_OUT, HIGH); speakingStartTime = millis(); - webSocket.enableHeartbeat(30000, 15000, 3); + // webSocket.enableHeartbeat(30000, 15000, 3); Serial.println("Transitioned to speaking mode"); } @@ -97,7 +107,7 @@ void transitionToListening() { deviceState = LISTENING; digitalWrite(I2S_SD_OUT, LOW); - webSocket.disableHeartbeat(); + // webSocket.disableHeartbeat(); } // audioStreamTask -> copier.copy() (conditional on webSocket.isConnected()) @@ -131,24 +141,34 @@ void audioStreamTask(void *parameter) { config.port_no = I2S_PORT_OUT; config.copyFrom(info); - i2s.begin(config); + i2s.begin(config); + // Initialize both volume streams once auto vcfg = volume.defaultConfig(); - vcfg.copyFrom(config); + vcfg.copyFrom(info); vcfg.allow_boost = true; volume.begin(vcfg); + + auto vcfgPitch = volumePitch.defaultConfig(); + vcfgPitch.copyFrom(info); + vcfgPitch.allow_boost = true; + volumePitch.begin(vcfgPitch); while (1) { if ( i2sOutputFlushScheduled) { i2sOutputFlushScheduled = false; i2s.flush(); volume.flush(); + volumePitch.flush(); queue.flush(); - //audioBuffer.reset(); todo: read untill empty } if (webSocket.isConnected() && deviceState == SPEAKING) { - copier.copy(); + if (currentPitchFactor != 1.0f) { + pitchCopier.copy(); + } else { + copier.copy(); + } } else { //we should always read from audioBuffer, otherwise writing thread can stuck @@ -260,9 +280,23 @@ void webSocketEvent(WStype_t type, uint8_t *payload, size_t length) // auth messages if (strcmp((char*)type.c_str(), "auth") == 0) { currentVolume = doc["volume_control"].as(); + currentPitchFactor = doc["pitch_factor"].as(); + bool is_ota = doc["is_ota"].as(); bool is_reset = doc["is_reset"].as(); - volume.setVolume(currentVolume / 100.0f); // Set initial volume (e.g., 70/100 = 0.7) + + // Update volumes on both streams + volume.setVolume(currentVolume / 100.0f); + volumePitch.setVolume(currentVolume / 100.0f); + + // Only initialize pitch shift if needed + if (currentPitchFactor != 1.0f) { + auto pcfg = pitchShift.defaultConfig(); + pcfg.copyFrom(info); + pcfg.pitch_shift = currentPitchFactor; + pcfg.buffer_size = 512; + pitchShift.begin(pcfg); + } if (is_ota) { Serial.println("OTA update received"); @@ -339,8 +373,9 @@ void websocketSetup(String server_domain, int port, String path) webSocket.setExtraHeaders(headers.c_str()); webSocket.onEvent(webSocketEvent); webSocket.setReconnectInterval(1000); + webSocket.disableHeartbeat(); - webSocket.enableHeartbeat(30000, 15000, 3); // 30s ping interval, 15s timeout, 3 retries + // webSocket.enableHeartbeat(30000, 15000, 3); // 30s ping interval, 15s timeout, 3 retries #ifdef DEV_MODE webSocket.begin(server_domain.c_str(), port, path.c_str()); @@ -366,4 +401,4 @@ void networkTask(void *parameter) { vTaskDelay(1); } -} +} \ No newline at end of file diff --git a/firmware-arduino/src/Audio.h b/firmware-arduino/src/Audio.h index c2463cd..f6d0c2f 100644 --- a/firmware-arduino/src/Audio.h +++ b/firmware-arduino/src/Audio.h @@ -30,6 +30,11 @@ extern I2SStream i2s; extern VolumeStream volume; extern QueueStream queue; extern StreamCopy copier; + +// NEW for pitch shift +extern VolumeStream volumePitch; +extern StreamCopy pitchCopier; + extern AudioInfo info; extern volatile bool i2sOutputFlushScheduled; @@ -48,4 +53,4 @@ unsigned long getSpeakingDuration(); void audioStreamTask(void *parameter); // AUDIO INPUT -void micTask(void *parameter); +void micTask(void *parameter); \ No newline at end of file diff --git a/firmware-arduino/src/Config.cpp b/firmware-arduino/src/Config.cpp index e5b3bcf..f8f60b7 100644 --- a/firmware-arduino/src/Config.cpp +++ b/firmware-arduino/src/Config.cpp @@ -32,11 +32,11 @@ bool factory_reset_status = false; */ #ifdef DEV_MODE -const char *ws_server = "10.2.1.136"; +const char *ws_server = "10.2.1.132"; const uint16_t ws_port = 8000; const char *ws_path = "/"; // Backend server details -const char *backend_server = "10.2.1.136"; +const char *backend_server = "10.2.1.132"; const uint16_t backend_port = 3000; #elif defined(PROD_MODE) diff --git a/firmware-arduino/src/PitchShift.cpp b/firmware-arduino/src/PitchShift.cpp new file mode 100644 index 0000000..a3a886f --- /dev/null +++ b/firmware-arduino/src/PitchShift.cpp @@ -0,0 +1,54 @@ +#include "PitchShift.h" + +#define GRAINSIZE 1024ul +static int16_t buf1[GRAINSIZE]; +static int16_t buf2[GRAINSIZE]; +static int16_t* buf = buf1; +static int16_t* buf_ = buf2; +static unsigned long readAddress = 0; +static unsigned long writeAddress = 0; + +bool PitchShiftFixedOutput::begin(PitchShiftInfo info) { + TRACED(); + cfg = info; + AudioOutput::setAudioInfo(info); + this->pitchMul = (uint32_t)(info.pitch_shift * 256.0f + 0.5f); + //this->secondaryOffset = (uint32_t)( (1.0f - (info.pitch_shift - (int)(info.pitch_shift))) * GRAINSIZE + 0.5f); + this->secondaryOffset = GRAINSIZE - ((( this->pitchMul * GRAINSIZE ) >> 8 ) % GRAINSIZE); + + return true; +} + +int16_t PitchShiftFixedOutput::pitchShift(int16_t value) { + + buf_[writeAddress] = value; + + int ii1 = (writeAddress * this->pitchMul) >> 8; + int output1 = buf[ii1 % GRAINSIZE]; + int ii2 = ii1 + secondaryOffset; + int output2 = buf[ii2 % GRAINSIZE]; + + unsigned long f = 0; + if ( writeAddress >= GRAINSIZE*3/4 ) + { + f = GRAINSIZE; + } + else if ( writeAddress >= GRAINSIZE/4 ) + { + f = (writeAddress - GRAINSIZE/4) * 2; + } + + int output = ( output1 * (GRAINSIZE - f ) + output2 * f ) /GRAINSIZE; + + writeAddress++; + if (writeAddress >= GRAINSIZE) + { + writeAddress = 0; // loop around to beginning of grain + readAddress = 0; + + buf_ = buf; + buf = buf == buf1 ? buf2 : buf1; + } + + return output; +} diff --git a/firmware-arduino/src/PitchShift.h b/firmware-arduino/src/PitchShift.h new file mode 100644 index 0000000..f3f2d4f --- /dev/null +++ b/firmware-arduino/src/PitchShift.h @@ -0,0 +1,44 @@ +#pragma once + +#include +#include +#include + +#include "AudioTools.h" + +//pitch shift effect with interpolaion fixed to 1.5 frequency factor, fixed delay, int16_t, 1 channel +class PitchShiftFixedOutput : public AudioOutput { +public: + PitchShiftFixedOutput(Print &out) { p_out = &out; } + + PitchShiftInfo defaultConfig() { + PitchShiftInfo result; + result.bits_per_sample = sizeof(int16_t) * 8; + return result; + } + + bool begin(PitchShiftInfo info); + + size_t write(const uint8_t *data, size_t len) override { + size_t result = 0; + int16_t *p_in = (int16_t *)data; + int sample_count = len / sizeof(int16_t); + + for (int j = 0; j < sample_count; j += 1) { + int16_t value = p_in[j]; + + int16_t out_value = pitchShift(value); + result += p_out->write((uint8_t *)&out_value, sizeof(int16_t)); + } + return result; + } + + void end() {} + +protected: + Print *p_out = nullptr; + + int16_t pitchShift(int16_t value); + uint32_t pitchMul; + unsigned long secondaryOffset; +}; \ No newline at end of file diff --git a/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx b/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx index 976edd8..9c0febf 100644 --- a/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx +++ b/frontend-nextjs/app/components/CreateCharacter/BuildDashboard.tsx @@ -17,6 +17,8 @@ import { useRouter } from "next/navigation"; import { z } from "zod"; import { emotionOptions, r2UrlAudio, voices } from "@/lib/data"; import EmojiComponent from "./EmojiComponent"; +import { PitchFactors } from "@/lib/utils"; +import { Slider } from "@/components/ui/slider"; interface SettingsDashboardProps { selectedUser: IUser; @@ -30,7 +32,8 @@ const formSchema = z.object({ voice: z.string().min(1, "Voice selection is required"), voiceCharacteristics: z.object({ features: z.string().min(10, "Minimum 10 characters").max(150, "Maximum 150 characters"), - emotion: z.string() + emotion: z.string(), + pitchFactor: z.number().min(0.75).max(1.5), }) }); @@ -57,7 +60,8 @@ const SettingsDashboard: React.FC = ({ voice: '', voiceCharacteristics: { features: '', - emotion: 'neutral' + emotion: 'neutral', + pitchFactor: 1.0, } }); @@ -110,30 +114,33 @@ const SettingsDashboard: React.FC = ({ } }; - const handleVoiceCharacteristicChange = (characteristic: 'features' | 'emotion', value: string) => { - const newVoiceCharacteristics = { - ...formData.voiceCharacteristics, - [characteristic]: value - }; - - // Validate just this nested field - try { - formSchema.shape.voiceCharacteristics.shape[characteristic].parse(value); - // Clear error if validation passes - setFormErrors(prev => ({ ...prev, [characteristic]: undefined })); - } catch (error: unknown) { - if (error instanceof z.ZodError) { - // Type assertion is needed here - const zodError = error as z.ZodError; - setFormErrors(prev => ({ ...prev, [characteristic]: zodError.errors[0].message })); + const handleVoiceCharacteristicChange = (characteristic: 'features' | 'emotion' | 'pitchFactor', value: string | number) => { + const newVoiceCharacteristics = { + ...formData.voiceCharacteristics, + [characteristic]: characteristic === 'pitchFactor' ? Number(value) : value + }; + + // Validate just this nested field + try { + if (characteristic === 'pitchFactor') { + formSchema.shape.voiceCharacteristics.shape.pitchFactor.parse(newVoiceCharacteristics[characteristic]); + } else { + formSchema.shape.voiceCharacteristics.shape[characteristic].parse(newVoiceCharacteristics[characteristic]); + } + // Clear error if validation passes + setFormErrors(prev => ({ ...prev, [characteristic]: undefined })); + } catch (error: unknown) { + if (error instanceof z.ZodError) { + const zodError = error as z.ZodError; + setFormErrors(prev => ({ ...prev, [characteristic]: zodError.errors[0].message })); + } } - } - - setFormData({ - ...formData, - voiceCharacteristics: newVoiceCharacteristics - }); - }; + + setFormData({ + ...formData, + voiceCharacteristics: newVoiceCharacteristics + }); + }; const handleSubmit = async (e: React.FormEvent) => { @@ -174,7 +181,8 @@ const SettingsDashboard: React.FC = ({ is_story: false, key: formData.title.toLowerCase().replace(/ /g, '_') + "_" + uuidv4(), creator_id: selectedUser.user_id, - short_description: formData.description + short_description: formData.description, + pitch_factor: formData.voiceCharacteristics.pitchFactor }); if (personality) { @@ -248,204 +256,240 @@ const SettingsDashboard: React.FC = ({ }; return ( -
- -
- - {currentStep === 'personality' ?
-

- Character Details -

-
- - handleInputChange('title', e.target.value)} - onBlur={() => handleBlur('title')} - /> -

- - {formErrors.title || "Give your AI character a name or title."} - - {formData.title.length}/50 -

-
-
- -