From 9556d07484769e3edeca9a5b19674d8040395a79 Mon Sep 17 00:00:00 2001 From: Rodrigo Date: Fri, 1 Dec 2023 17:33:46 -0300 Subject: [PATCH] vad --- mic_test_whisper_simple.py | 95 +++++++++++++++++++++++++++ mic_test_whisper_streaming.py | 71 +++++++++++++++++++++ microphone_stream.py | 82 ++++++++++++++++++++++++ voice_activity_controller.py | 117 ++++++++++++++++++++++++++++++++++ 4 files changed, 365 insertions(+) create mode 100644 mic_test_whisper_simple.py create mode 100644 mic_test_whisper_streaming.py create mode 100644 microphone_stream.py create mode 100644 voice_activity_controller.py diff --git a/mic_test_whisper_simple.py b/mic_test_whisper_simple.py new file mode 100644 index 0000000..58d3a8d --- /dev/null +++ b/mic_test_whisper_simple.py @@ -0,0 +1,95 @@ +from microphone_stream import MicrophoneStream +from voice_activity_controller import VoiceActivityController +from whisper_online import * +import numpy as np +import librosa +import io +import soundfile +import sys + + + + +class SimpleASRProcessor: + + def __init__(self, asr, sampling_rate = 16000): + """run this when starting or restarting processing""" + self.audio_buffer = np.array([],dtype=np.float32) + self.prompt_buffer = "" + self.asr = asr + self.sampling_rate = sampling_rate + self.init_prompt = '' + + def ts_words(self, segments): + result = "" + for segment in segments: + if segment.no_speech_prob > 0.9: + continue + for word in segment.words: + w = word.word + t = (word.start, word.end, w) + result +=w + return result + + def stream_process(self, vad_result): + iter_in_phrase = 0 + for chunk, is_final in vad_result: + iter_in_phrase += 1 + + if chunk is not None: + sf = soundfile.SoundFile(io.BytesIO(chunk), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") + audio, _ = librosa.load(sf,sr=SAMPLING_RATE) + # self.audio_buffer.append(chunk) + out = [] + out.append(audio) + a = np.concatenate(out) + self.audio_buffer = np.append(self.audio_buffer, a) + + if is_final and len(self.audio_buffer) > 0: + res = self.asr.transcribe(self.audio_buffer, init_prompt=self.init_prompt) + # use custom ts_words + tsw = self.ts_words(res) + self.init_prompt = self.init_prompt + tsw + self.init_prompt = self.init_prompt [-100:] + self.audio_buffer.resize(0) + iter_in_phrase =0 + yield True, tsw + # show progress evry 10 chunks + elif iter_in_phrase % 20 == 0 and len(self.audio_buffer) > 0: + res = self.asr.transcribe(self.audio_buffer, init_prompt=self.init_prompt) + # use custom ts_words + tsw = self.ts_words(res) + yield False, tsw + + + + + + + +SAMPLING_RATE = 16000 + +model = "large-v2" +src_lan = "en" # source language +tgt_lan = "en" # target language -- same as source for ASR, "en" if translate task is used +use_vad_result = True +min_sample_length = 1 * SAMPLING_RATE + + + +vad = VoiceActivityController(use_vad_result = use_vad_result) +asr = FasterWhisperASR(src_lan, "large-v2") # loads and wraps Whisper model + +tokenizer = create_tokenizer(tgt_lan) +online = SimpleASRProcessor(asr) + + +stream = MicrophoneStream() +stream = vad.detect_user_speech(stream, audio_in_int16 = False) +stream = online.stream_process(stream) + +for isFinal, text in stream: + if isFinal: + print( text, end="\r\n") + else: + print( text, end="\r") diff --git a/mic_test_whisper_streaming.py b/mic_test_whisper_streaming.py new file mode 100644 index 0000000..26c0ba5 --- /dev/null +++ b/mic_test_whisper_streaming.py @@ -0,0 +1,71 @@ +from microphone_stream import MicrophoneStream +from voice_activity_controller import VoiceActivityController +from whisper_online import * +import numpy as np +import librosa +import io +import soundfile +import sys + + +SAMPLING_RATE = 16000 +model = "large-v2" +src_lan = "en" # source language +tgt_lan = "en" # target language -- same as source for ASR, "en" if translate task is used +use_vad_result = True +min_sample_length = 1 * SAMPLING_RATE + + + +asr = FasterWhisperASR(src_lan, model) # loads and wraps Whisper model +tokenizer = create_tokenizer(tgt_lan) # sentence segmenter for the target language +online = OnlineASRProcessor(asr, tokenizer) # create processing object + +microphone_stream = MicrophoneStream() +vad = VoiceActivityController(use_vad_result = use_vad_result) + +complete_text = '' +final_processing_pending = False +out = [] +out_len = 0 +for iter in vad.detect_user_speech(microphone_stream): # processing loop: + raw_bytes= iter[0] + is_final = iter[1] + + if raw_bytes: + sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") + audio, _ = librosa.load(sf,sr=SAMPLING_RATE) + out.append(audio) + out_len += len(audio) + + + if (is_final or out_len >= min_sample_length) and out_len>0: + a = np.concatenate(out) + online.insert_audio_chunk(a) + + if out_len > min_sample_length: + o = online.process_iter() + print('-----'*10) + complete_text = complete_text + o[2] + print('PARTIAL - '+ complete_text) # do something with current partial output + print('-----'*10) + out = [] + out_len = 0 + + if is_final: + o = online.finish() + online.init() + # final_processing_pending = False + print('-----'*10) + complete_text = complete_text + o[2] + print('FINAL - '+ complete_text) # do something with current partial output + print('-----'*10) + out = [] + out_len = 0 + + + + + + + diff --git a/microphone_stream.py b/microphone_stream.py new file mode 100644 index 0000000..c317844 --- /dev/null +++ b/microphone_stream.py @@ -0,0 +1,82 @@ + + +### mic stream + +import queue +import re +import sys +import pyaudio + + +class MicrophoneStream: + def __init__( + self, + sample_rate: int = 16000, + ): + """ + Creates a stream of audio from the microphone. + + Args: + chunk_size: The size of each chunk of audio to read from the microphone. + channels: The number of channels to record audio from. + sample_rate: The sample rate to record audio at. + """ + try: + import pyaudio + except ImportError: + raise Exception('py audio not installed') + + self._pyaudio = pyaudio.PyAudio() + self.sample_rate = sample_rate + + self._chunk_size = int(self.sample_rate * 0.1) + self._stream = self._pyaudio.open( + format=pyaudio.paInt16, + channels=1, + rate=sample_rate, + input=True, + frames_per_buffer=self._chunk_size, + ) + + self._open = True + + def __iter__(self): + """ + Returns the iterator object. + """ + + return self + + def __next__(self): + """ + Reads a chunk of audio from the microphone. + """ + if not self._open: + raise StopIteration + + try: + return self._stream.read(self._chunk_size) + except KeyboardInterrupt: + raise StopIteration + + def close(self): + """ + Closes the stream. + """ + + self._open = False + + if self._stream.is_active(): + self._stream.stop_stream() + + self._stream.close() + self._pyaudio.terminate() + + + + + + + + + diff --git a/voice_activity_controller.py b/voice_activity_controller.py new file mode 100644 index 0000000..d1cf031 --- /dev/null +++ b/voice_activity_controller.py @@ -0,0 +1,117 @@ +import torch +import numpy as np +# import sounddevice as sd +import torch +import numpy as np + + +class VoiceActivityController: + def __init__( + self, + sampling_rate = 16000, + second_ofSilence = 0.5, + second_ofSpeech = 0.25, + second_ofMinRecording = 10, + use_vad_result = True, + activity_detected_callback=None, + ): + self.activity_detected_callback=activity_detected_callback + self.model, self.utils = torch.hub.load( + repo_or_dir='snakers4/silero-vad', + model='silero_vad' + ) + (self.get_speech_timestamps, + save_audio, + read_audio, + VADIterator, + collect_chunks) = self.utils + + self.sampling_rate = sampling_rate + self.silence_limit = second_ofSilence * self.sampling_rate + self.speech_limit = second_ofSpeech *self.sampling_rate + self.MIN_RECORDING_LENGTH = second_ofMinRecording * self.sampling_rate + + self.use_vad_result = use_vad_result + self.vad_iterator = VADIterator( + model =self.model, + threshold = 0.3, + sampling_rate= 16000, + min_silence_duration_ms = 500, #100 + speech_pad_ms = 400 #30 + ) + self.last_marked_chunk = None + + + def int2float(self, sound): + abs_max = np.abs(sound).max() + sound = sound.astype('float32') + if abs_max > 0: + sound *= 1/32768 + sound = sound.squeeze() # depends on the use case + return sound + + def apply_vad(self, audio): + audio_float32 = self.int2float(audio) + chunk = self.vad_iterator(audio_float32, return_seconds=False) + + if chunk is not None: + if "start" in chunk: + start = chunk["start"] + self.last_marked_chunk = chunk + return audio[start:] if self.use_vad_result else audio, (len(audio) - start), 0 + + if "end" in chunk: + #todo: pending get the padding from the next chunk + end = chunk["end"] if chunk["end"] < len(audio) else len(audio) + self.last_marked_chunk = chunk + return audio[:end] if self.use_vad_result else audio, end ,len(audio) - end + + if self.last_marked_chunk is not None: + if "start" in self.last_marked_chunk: + return audio, len(audio) ,0 + + if "end" in self.last_marked_chunk: + return np.array([], dtype=np.float16) if self.use_vad_result else audio, 0 ,len(audio) + + return np.array([], dtype=np.float16) if self.use_vad_result else audio, 0 , 0 + + + + def detect_user_speech(self, audio_stream, audio_in_int16 = False): + silence_len= 0 + speech_len = 0 + + for data in audio_stream: # replace with your condition of choice + # if isinstance(data, EndOfTransmission): + # raise EndOfTransmission("End of transmission detected") + + + audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data + wav = audio_block + + + is_final = False + voice_audio, speech_in_wav, last_silent_duration_in_wav = self.apply_vad(wav) + # print(f'----r> speech_in_wav: {speech_in_wav} last_silent_duration_in_wav: {last_silent_duration_in_wav}') + + if speech_in_wav > 0 : + silence_len= 0 + speech_len += speech_in_wav + if self.activity_detected_callback is not None: + self.activity_detected_callback() + + silence_len = silence_len + last_silent_duration_in_wav + if silence_len>= self.silence_limit and speech_len >= self.speech_limit: + is_final = True + silence_len= 0 + speech_len = 0 + + + yield voice_audio.tobytes(), is_final + + + + + + +