admin-valentin/WhisperLiveKit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Further tidying of print output, so by default there's little on the console

Browse source
This commit is contained in:
Alex Young 2024-04-14 19:14:56 +01:00
parent 5ebbed3bd7
commit 380c30d48d
2 changed files with 42 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
whisper_online.py
View file

@ -4,6 +4,7 @@ import numpy as np
import librosa import librosa
from functools import lru_cache from functools import lru_cache
import time import time
import logging
@ -57,7 +58,7 @@ class WhisperTimestampedASR(ASRBase):
from whisper_timestamped import transcribe_timestamped from whisper_timestamped import transcribe_timestamped
self.transcribe_timestamped = transcribe_timestamped self.transcribe_timestamped = transcribe_timestamped
if model_dir is not None: if model_dir is not None:
print("ignoring model_dir, not implemented",file=self.logfile) logging.debug("ignoring model_dir, not implemented")
return whisper.load_model(modelsize, download_root=cache_dir) return whisper.load_model(modelsize, download_root=cache_dir)
def transcribe(self, audio, init_prompt=""): def transcribe(self, audio, init_prompt=""):
@ -97,7 +98,7 @@ class FasterWhisperASR(ASRBase):
def load_model(self, modelsize=None, cache_dir=None, model_dir=None): def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
from faster_whisper import WhisperModel from faster_whisper import WhisperModel
if model_dir is not None: if model_dir is not None:
print(f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used.",file=self.logfile) logging.debug(f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used.")
model_size_or_path = model_dir model_size_or_path = model_dir
elif modelsize is not None: elif modelsize is not None:
model_size_or_path = modelsize model_size_or_path = modelsize
@ -173,9 +174,11 @@ class HypothesisBuffer:
c = " ".join([self.commited_in_buffer[-j][2] for j in range(1,i+1)][::-1]) c = " ".join([self.commited_in_buffer[-j][2] for j in range(1,i+1)][::-1])
tail = " ".join(self.new[j-1][2] for j in range(1,i+1)) tail = " ".join(self.new[j-1][2] for j in range(1,i+1))
if c == tail: if c == tail:
print("removing last",i,"words:",file=self.logfile) words = []
for j in range(i): for j in range(i):
print("\t",self.new.pop(0),file=self.logfile) words.append(repr(self.new.pop(0)))
words_msg = "\t".join(words)
logging.debug(f"removing last {i} words: {words_msg}")
break break
def flush(self): def flush(self):
@ -267,9 +270,9 @@ class OnlineASRProcessor:
""" """
prompt, non_prompt = self.prompt() prompt, non_prompt = self.prompt()
print("PROMPT:", prompt, file=self.logfile) logging.debug(f"PROMPT: {prompt}")
print("CONTEXT:", non_prompt, file=self.logfile) logging.debug(f"CONTEXT: {non_prompt}")
print(f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}",file=self.logfile) logging.debug(f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}")
res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt) res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt)
# transform to [(beg,end,"word1"), ...] # transform to [(beg,end,"word1"), ...]
@ -278,8 +281,10 @@ class OnlineASRProcessor:
self.transcript_buffer.insert(tsw, self.buffer_time_offset) self.transcript_buffer.insert(tsw, self.buffer_time_offset)
o = self.transcript_buffer.flush() o = self.transcript_buffer.flush()
self.commited.extend(o) self.commited.extend(o)
print(">>>>COMPLETE NOW:",self.to_flush(o),file=self.logfile,flush=True) completed = self.to_flush(o)
print("INCOMPLETE:",self.to_flush(self.transcript_buffer.complete()),file=self.logfile,flush=True) logging.debug(f">>>>COMPLETE NOW: {completed}")
the_rest = self.to_flush(self.transcript_buffer.complete())
logging.debug(f"INCOMPLETE: {the_rest}")
# there is a newly confirmed text # there is a newly confirmed text
@ -303,18 +308,18 @@ class OnlineASRProcessor:
#while k>0 and self.commited[k][1] > l: #while k>0 and self.commited[k][1] > l:
# k -= 1 # k -= 1
#t = self.commited[k][1] #t = self.commited[k][1]
print(f"chunking segment",file=self.logfile) logging.debug(f"chunking segment")
#self.chunk_at(t) #self.chunk_at(t)
print(f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}",file=self.logfile) logging.debug(f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}")
return self.to_flush(o) return self.to_flush(o)
def chunk_completed_sentence(self): def chunk_completed_sentence(self):
if self.commited == []: return if self.commited == []: return
print(self.commited,file=self.logfile) logging.debug(self.commited)
sents = self.words_to_sentences(self.commited) sents = self.words_to_sentences(self.commited)
for s in sents: for s in sents:
print("\t\tSENT:",s,file=self.logfile) logging.debug(f"\t\tSENT: {s}")
if len(sents) < 2: if len(sents) < 2:
return return
while len(sents) > 2: while len(sents) > 2:
@ -322,7 +327,7 @@ class OnlineASRProcessor:
# we will continue with audio processing at this timestamp # we will continue with audio processing at this timestamp
chunk_at = sents[-2][1] chunk_at = sents[-2][1]
print(f"--- sentence chunked at {chunk_at:2.2f}",file=self.logfile) logging.debug(f"--- sentence chunked at {chunk_at:2.2f}")
self.chunk_at(chunk_at) self.chunk_at(chunk_at)
def chunk_completed_segment(self, res): def chunk_completed_segment(self, res):
@ -339,12 +344,12 @@ class OnlineASRProcessor:
ends.pop(-1) ends.pop(-1)
e = ends[-2]+self.buffer_time_offset e = ends[-2]+self.buffer_time_offset
if e <= t: if e <= t:
print(f"--- segment chunked at {e:2.2f}",file=self.logfile) logging.debug(f"--- segment chunked at {e:2.2f}")
self.chunk_at(e) self.chunk_at(e)
else: else:
print(f"--- last segment not within commited area",file=self.logfile) logging.debug(f"--- last segment not within commited area")
else: else:
print(f"--- not enough segments to chunk",file=self.logfile) logging.debug(f"--- not enough segments to chunk")
@ -391,7 +396,7 @@ class OnlineASRProcessor:
""" """
o = self.transcript_buffer.complete() o = self.transcript_buffer.complete()
f = self.to_flush(o) f = self.to_flush(o)
print("last, noncommited:",f,file=self.logfile) logging.debug("last, noncommited: {f}")
return f return f
@ -431,7 +436,7 @@ def create_tokenizer(lan):
# the following languages are in Whisper, but not in wtpsplit: # the following languages are in Whisper, but not in wtpsplit:
if lan in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split(): if lan in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split():
print(f"{lan} code is not supported by wtpsplit. Going to use None lang_code option.", file=sys.stderr) logging.debug(f"{lan} code is not supported by wtpsplit. Going to use None lang_code option.")
lan = None lan = None
from wtpsplit import WtP from wtpsplit import WtP
@ -476,20 +481,20 @@ if __name__ == "__main__":
logfile = sys.stderr logfile = sys.stderr
if args.offline and args.comp_unaware: if args.offline and args.comp_unaware:
print("No or one option from --offline and --comp_unaware are available, not both. Exiting.",file=logfile) logging.error("No or one option from --offline and --comp_unaware are available, not both. Exiting.")
sys.exit(1) sys.exit(1)
audio_path = args.audio_path audio_path = args.audio_path
SAMPLING_RATE = 16000 SAMPLING_RATE = 16000
duration = len(load_audio(audio_path))/SAMPLING_RATE duration = len(load_audio(audio_path))/SAMPLING_RATE
print("Audio duration is: %2.2f seconds" % duration, file=logfile) logging.info("Audio duration is: %2.2f seconds" % duration)
size = args.model size = args.model
language = args.lan language = args.lan
t = time.time() t = time.time()
print(f"Loading Whisper {size} model for {language}...",file=logfile,end=" ",flush=True) logging.info(f"Loading Whisper {size} model for {language}...")
if args.backend == "faster-whisper": if args.backend == "faster-whisper":
asr_cls = FasterWhisperASR asr_cls = FasterWhisperASR
@ -506,10 +511,10 @@ if __name__ == "__main__":
e = time.time() e = time.time()
print(f"done. It took {round(e-t,2)} seconds.",file=logfile) logging.info(f"done. It took {round(e-t,2)} seconds.")
if args.vad: if args.vad:
print("setting VAD filter",file=logfile) logging.info("setting VAD filter")
asr.use_vad() asr.use_vad()
@ -543,16 +548,15 @@ if __name__ == "__main__":
print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True) print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True)
print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True) print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True)
else: else:
print(o,file=logfile,flush=True) print("here?", o,file=logfile,flush=True)
if args.offline: ## offline mode processing (for testing/debugging) if args.offline: ## offline mode processing (for testing/debugging)
a = load_audio(audio_path) a = load_audio(audio_path)
online.insert_audio_chunk(a) online.insert_audio_chunk(a)
try: try:
o = online.process_iter() o = online.process_iter()
except AssertionError: except AssertionError as e:
print("assertion error",file=logfile) log.error(f"assertion error: {repr(e)}")
pass
else: else:
output_transcript(o) output_transcript(o)
now = None now = None
@ -563,13 +567,13 @@ if __name__ == "__main__":
online.insert_audio_chunk(a) online.insert_audio_chunk(a)
try: try:
o = online.process_iter() o = online.process_iter()
except AssertionError: except AssertionError as e:
print("assertion error",file=logfile) logging.error(f"assertion error: {repr(e)}")
pass pass
else: else:
output_transcript(o, now=end) output_transcript(o, now=end)
print(f"## last processed {end:.2f}s",file=logfile,flush=True) logging.debug(f"## last processed {end:.2f}s")
if end >= duration: if end >= duration:
break break
@ -595,13 +599,13 @@ if __name__ == "__main__":
try: try:
o = online.process_iter() o = online.process_iter()
except AssertionError: except AssertionError as e:
print("assertion error",file=logfile) logging.error(f"assertion error: {e}")
pass pass
else: else:
output_transcript(o) output_transcript(o)
now = time.time() - start now = time.time() - start
print(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}",file=logfile,flush=True) logging.debug(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}")
if end >= duration: if end >= duration:
break break

10
whisper_online_server.py
View file

@ -39,6 +39,7 @@ logging.debug(f"Loading Whisper {size} model for {language}...")
if args.backend == "faster-whisper": if args.backend == "faster-whisper":
from faster_whisper import WhisperModel from faster_whisper import WhisperModel
asr_cls = FasterWhisperASR asr_cls = FasterWhisperASR
logging.getLogger("faster_whisper").setLevel(logging.WARNING)
else: else:
import whisper import whisper
import whisper_timestamped import whisper_timestamped
@ -80,7 +81,7 @@ if os.path.exists(demo_audio_path):
# warm up the ASR, because the very first transcribe takes much more time than the other # warm up the ASR, because the very first transcribe takes much more time than the other
asr.transcribe(a) asr.transcribe(a)
else: else:
logging.info("Whisper is not warmed up") logging.debug("Whisper is not warmed up")
######### Server objects ######### Server objects
@ -135,8 +136,6 @@ class ServerProcessor:
out = [] out = []
while sum(len(x) for x in out) < self.min_chunk*SAMPLING_RATE: while sum(len(x) for x in out) < self.min_chunk*SAMPLING_RATE:
raw_bytes = self.connection.non_blocking_receive_audio() raw_bytes = self.connection.non_blocking_receive_audio()
print(raw_bytes[:10])
print(len(raw_bytes))
if not raw_bytes: if not raw_bytes:
break break
sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW") sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW")
@ -167,7 +166,7 @@ class ServerProcessor:
print("%1.0f %1.0f %s" % (beg,end,o[2]),flush=True,file=sys.stderr) print("%1.0f %1.0f %s" % (beg,end,o[2]),flush=True,file=sys.stderr)
return "%1.0f %1.0f %s" % (beg,end,o[2]) return "%1.0f %1.0f %s" % (beg,end,o[2])
else: else:
print(o,file=sys.stderr,flush=True) # No text, so no output
return None return None
def send_result(self, o): def send_result(self, o):
@ -181,14 +180,13 @@ class ServerProcessor:
while True: while True:
a = self.receive_audio_chunk() a = self.receive_audio_chunk()
if a is None: if a is None:
print("break here",file=sys.stderr)
break break
self.online_asr_proc.insert_audio_chunk(a) self.online_asr_proc.insert_audio_chunk(a)
o = online.process_iter() o = online.process_iter()
try: try:
self.send_result(o) self.send_result(o)
except BrokenPipeError: except BrokenPipeError:
print("broken pipe -- connection closed?",file=sys.stderr) logging.info("broken pipe -- connection closed?")
break break
# o = online.finish() # this should be working # o = online.finish() # this should be working