admin-valentin/WhisperLiveKit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

black formating

Browse source
This commit is contained in:
silask 2024-12-30 21:20:38 +01:00
parent 4cb3660666
commit 5fdb08edae
3 changed files with 492 additions and 244 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
silero_vad_iterator.py
View file

@ -6,15 +6,16 @@ import torch
# Their licence is MIT, same as ours: https://github.com/snakers4/silero-vad/blob/f6b1294cb27590fb2452899df98fb234dfef1134/LICENSE # Their licence is MIT, same as ours: https://github.com/snakers4/silero-vad/blob/f6b1294cb27590fb2452899df98fb234dfef1134/LICENSE
class VADIterator: class VADIterator:
def __init__(self, def __init__(
self,
model, model,
threshold: float = 0.5, threshold: float = 0.5,
sampling_rate: int = 16000, sampling_rate: int = 16000,
min_silence_duration_ms: int = 500, # makes sense on one recording that I checked min_silence_duration_ms: int = 500, # makes sense on one recording that I checked
speech_pad_ms: int = 100 # same speech_pad_ms: int = 100, # same
): ):
""" """
Class for stream imitation Class for stream imitation
@ -41,7 +42,9 @@ class VADIterator:
self.sampling_rate = sampling_rate self.sampling_rate = sampling_rate
if sampling_rate not in [8000, 16000]: if sampling_rate not in [8000, 16000]:
raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]') raise ValueError(
"VADIterator does not support sampling rates other than [8000, 16000]"
)
self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000 self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000 self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
@ -80,7 +83,13 @@ class VADIterator:
if (speech_prob >= self.threshold) and not self.triggered: if (speech_prob >= self.threshold) and not self.triggered:
self.triggered = True self.triggered = True
speech_start = self.current_sample - self.speech_pad_samples speech_start = self.current_sample - self.speech_pad_samples
return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)} return {
"start": (
int(speech_start)
if not return_seconds
else round(speech_start / self.sampling_rate, 1)
)
}
if (speech_prob < self.threshold - 0.15) and self.triggered: if (speech_prob < self.threshold - 0.15) and self.triggered:
if not self.temp_end: if not self.temp_end:
@ -91,23 +100,32 @@ class VADIterator:
speech_end = self.temp_end + self.speech_pad_samples speech_end = self.temp_end + self.speech_pad_samples
self.temp_end = 0 self.temp_end = 0
self.triggered = False self.triggered = False
return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)} return {
"end": (
int(speech_end)
if not return_seconds
else round(speech_end / self.sampling_rate, 1)
)
}
return None return None
####################### #######################
# because Silero now requires exactly 512-sized audio chunks # because Silero now requires exactly 512-sized audio chunks
import numpy as np import numpy as np
class FixedVADIterator(VADIterator): class FixedVADIterator(VADIterator):
'''It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once. """It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once.
If audio to be processed at once is long and multiple voiced segments detected, If audio to be processed at once is long and multiple voiced segments detected,
then __call__ returns the start of the first segment, and end (or middle, which means no end) of the last segment. then __call__ returns the start of the first segment, and end (or middle, which means no end) of the last segment.
''' """
def reset_states(self): def reset_states(self):
super().reset_states() super().reset_states()
self.buffer = np.array([],dtype=np.float32) self.buffer = np.array([], dtype=np.float32)
def __call__(self, x, return_seconds=False): def __call__(self, x, return_seconds=False):
self.buffer = np.append(self.buffer, x) self.buffer = np.append(self.buffer, x)
@ -118,29 +136,28 @@ class FixedVADIterator(VADIterator):
if ret is None: if ret is None:
ret = r ret = r
elif r is not None: elif r is not None:
if 'end' in r: if "end" in r:
ret['end'] = r['end'] # the latter end ret["end"] = r["end"] # the latter end
if 'start' in r and 'end' in ret: # there is an earlier start. if "start" in r and "end" in ret: # there is an earlier start.
# Remove end, merging this segment with the previous one. # Remove end, merging this segment with the previous one.
del ret['end'] del ret["end"]
return ret if ret != {} else None return ret if ret != {} else None
if __name__ == "__main__": if __name__ == "__main__":
# test/demonstrate the need for FixedVADIterator: # test/demonstrate the need for FixedVADIterator:
import torch import torch
model, _ = torch.hub.load(
repo_or_dir='snakers4/silero-vad', model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")
model='silero_vad'
)
vac = FixedVADIterator(model) vac = FixedVADIterator(model)
# vac = VADIterator(model) # the second case crashes with this # vac = VADIterator(model) # the second case crashes with this
# this works: for both # this works: for both
audio_buffer = np.array([0]*(512),dtype=np.float32) audio_buffer = np.array([0] * (512), dtype=np.float32)
vac(audio_buffer) vac(audio_buffer)
# this crashes on the non FixedVADIterator with # this crashes on the non FixedVADIterator with
# ops.prim.RaiseException("Input audio chunk is too short", "builtins.ValueError") # ops.prim.RaiseException("Input audio chunk is too short", "builtins.ValueError")
audio_buffer = np.array([0]*(512-1),dtype=np.float32) audio_buffer = np.array([0] * (512 - 1), dtype=np.float32)
vac(audio_buffer) vac(audio_buffer)

74
whisper_fastapi_online_server.py
View file

@ -22,10 +22,21 @@ app.add_middleware(
parser = argparse.ArgumentParser(description="Whisper FastAPI Online Server") parser = argparse.ArgumentParser(description="Whisper FastAPI Online Server")
parser.add_argument("--host", type=str, default='localhost', help="The host address to bind the server to.") parser.add_argument(
parser.add_argument("--port", type=int, default=8000, help="The port number to bind the server to.") "--host",
parser.add_argument("--warmup-file", type=str, dest="warmup_file", type=str,
help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .") default="localhost",
help="The host address to bind the server to.",
)
parser.add_argument(
"--port", type=int, default=8000, help="The port number to bind the server to."
)
parser.add_argument(
"--warmup-file",
type=str,
dest="warmup_file",
help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .",
)
add_shared_args(parser) add_shared_args(parser)
args = parser.parse_args() args = parser.parse_args()
@ -35,29 +46,38 @@ asr, online = asr_factory(args)
with open("src/live_transcription.html", "r") as f: with open("src/live_transcription.html", "r") as f:
html = f.read() html = f.read()
@app.get("/") @app.get("/")
async def get(): async def get():
return HTMLResponse(html) return HTMLResponse(html)
SAMPLE_RATE = 16000 SAMPLE_RATE = 16000
CHANNELS = 1 CHANNELS = 1
SAMPLES_PER_SEC = SAMPLE_RATE * int(args.min_chunk_size) SAMPLES_PER_SEC = SAMPLE_RATE * int(args.min_chunk_size)
BYTES_PER_SAMPLE = 2 # s16le = 2 bytes per sample BYTES_PER_SAMPLE = 2 # s16le = 2 bytes per sample
BYTES_PER_SEC = SAMPLES_PER_SEC * BYTES_PER_SAMPLE BYTES_PER_SEC = SAMPLES_PER_SEC * BYTES_PER_SAMPLE
async def start_ffmpeg_decoder(): async def start_ffmpeg_decoder():
""" """
Start an FFmpeg process in async streaming mode that reads WebM from stdin Start an FFmpeg process in async streaming mode that reads WebM from stdin
and outputs raw s16le PCM on stdout. Returns the process object. and outputs raw s16le PCM on stdout. Returns the process object.
""" """
process = ( process = (
ffmpeg ffmpeg.input("pipe:0", format="webm")
.input('pipe:0', format='webm') .output(
.output('pipe:1', format='s16le', acodec='pcm_s16le', ac=CHANNELS, ar=str(SAMPLE_RATE)) "pipe:1",
format="s16le",
acodec="pcm_s16le",
ac=CHANNELS,
ar=str(SAMPLE_RATE),
)
.run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True) .run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
) )
return process return process
@app.websocket("/asr") @app.websocket("/asr")
async def websocket_endpoint(websocket: WebSocket): async def websocket_endpoint(websocket: WebSocket):
await websocket.accept() await websocket.accept()
@ -65,6 +85,7 @@ async def websocket_endpoint(websocket: WebSocket):
ffmpeg_process = await start_ffmpeg_decoder() ffmpeg_process = await start_ffmpeg_decoder()
pcm_buffer = bytearray() pcm_buffer = bytearray()
# Continuously read decoded PCM from ffmpeg stdout in a background task # Continuously read decoded PCM from ffmpeg stdout in a background task
async def ffmpeg_stdout_reader(): async def ffmpeg_stdout_reader():
nonlocal pcm_buffer nonlocal pcm_buffer
@ -75,9 +96,15 @@ async def websocket_endpoint(websocket: WebSocket):
try: try:
elapsed_time = int(time() - beg) elapsed_time = int(time() - beg)
beg = time() beg = time()
chunk = await loop.run_in_executor(None, ffmpeg_process.stdout.read, 32000*elapsed_time) chunk = await loop.run_in_executor(
if not chunk: # The first chunk will be almost empty, FFmpeg is still starting up None, ffmpeg_process.stdout.read, 32000 * elapsed_time
chunk = await loop.run_in_executor(None, ffmpeg_process.stdout.read, 4096) )
if (
not chunk
): # The first chunk will be almost empty, FFmpeg is still starting up
chunk = await loop.run_in_executor(
None, ffmpeg_process.stdout.read, 4096
)
if not chunk: # FFmpeg might have closed if not chunk: # FFmpeg might have closed
print("FFmpeg stdout closed.") print("FFmpeg stdout closed.")
break break
@ -86,21 +113,29 @@ async def websocket_endpoint(websocket: WebSocket):
if len(pcm_buffer) >= BYTES_PER_SEC: if len(pcm_buffer) >= BYTES_PER_SEC:
# Convert int16 -> float32 # Convert int16 -> float32
pcm_array = np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32) / 32768.0 pcm_array = (
np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32)
/ 32768.0
)
pcm_buffer = bytearray() pcm_buffer = bytearray()
online.insert_audio_chunk(pcm_array) online.insert_audio_chunk(pcm_array)
transcription = online.process_iter()[2] transcription = online.process_iter()[2]
full_transcription += transcription full_transcription += transcription
if args.vac: if args.vac:
buffer = online.online.to_flush(online.online.transcript_buffer.buffer)[2] # We need to access the underlying online object to get the buffer buffer = online.online.to_flush(
online.online.transcript_buffer.buffer
)[
2
] # We need to access the underlying online object to get the buffer
else: else:
buffer = online.to_flush(online.transcript_buffer.buffer)[2] buffer = online.to_flush(online.transcript_buffer.buffer)[2]
if buffer in full_transcription: # With VAC, the buffer is not updated until the next chunk is processed if (
buffer in full_transcription
): # With VAC, the buffer is not updated until the next chunk is processed
buffer = "" buffer = ""
await websocket.send_json({ await websocket.send_json(
"transcription": transcription, {"transcription": transcription, "buffer": buffer}
"buffer": buffer )
})
except Exception as e: except Exception as e:
print(f"Exception in ffmpeg_stdout_reader: {e}") print(f"Exception in ffmpeg_stdout_reader: {e}")
break break
@ -139,4 +174,7 @@ async def websocket_endpoint(websocket: WebSocket):
if __name__ == "__main__": if __name__ == "__main__":
import uvicorn import uvicorn
uvicorn.run("whisper_fastapi_online_server:app", host=args.host, port=args.port, reload=True)
uvicorn.run(
"whisper_fastapi_online_server:app", host=args.host, port=args.port, reload=True
)

503
whisper_online.py
View file

@ -12,26 +12,31 @@ import math
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@lru_cache(10**6) @lru_cache(10**6)
def load_audio(fname): def load_audio(fname):
a, _ = librosa.load(fname, sr=16000, dtype=np.float32) a, _ = librosa.load(fname, sr=16000, dtype=np.float32)
return a return a
def load_audio_chunk(fname, beg, end): def load_audio_chunk(fname, beg, end):
audio = load_audio(fname) audio = load_audio(fname)
beg_s = int(beg*16000) beg_s = int(beg * 16000)
end_s = int(end*16000) end_s = int(end * 16000)
return audio[beg_s:end_s] return audio[beg_s:end_s]
# Whisper backend # Whisper backend
class ASRBase: class ASRBase:
sep = " " # join transcribe words with this character (" " for whisper_timestamped, sep = " " # join transcribe words with this character (" " for whisper_timestamped,
# "" for faster-whisper because it emits the spaces when neeeded) # "" for faster-whisper because it emits the spaces when neeeded)
def __init__(self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr): def __init__(
self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr
):
self.logfile = logfile self.logfile = logfile
self.transcribe_kargs = {} self.transcribe_kargs = {}
@ -42,7 +47,6 @@ class ASRBase:
self.model = self.load_model(modelsize, cache_dir, model_dir) self.model = self.load_model(modelsize, cache_dir, model_dir)
def load_model(self, modelsize, cache_dir): def load_model(self, modelsize, cache_dir):
raise NotImplemented("must be implemented in the child class") raise NotImplemented("must be implemented in the child class")
@ -64,24 +68,30 @@ class WhisperTimestampedASR(ASRBase):
import whisper import whisper
import whisper_timestamped import whisper_timestamped
from whisper_timestamped import transcribe_timestamped from whisper_timestamped import transcribe_timestamped
self.transcribe_timestamped = transcribe_timestamped self.transcribe_timestamped = transcribe_timestamped
if model_dir is not None: if model_dir is not None:
logger.debug("ignoring model_dir, not implemented") logger.debug("ignoring model_dir, not implemented")
return whisper.load_model(modelsize, download_root=cache_dir) return whisper.load_model(modelsize, download_root=cache_dir)
def transcribe(self, audio, init_prompt=""): def transcribe(self, audio, init_prompt=""):
result = self.transcribe_timestamped(self.model, result = self.transcribe_timestamped(
audio, language=self.original_language, self.model,
initial_prompt=init_prompt, verbose=None, audio,
condition_on_previous_text=True, **self.transcribe_kargs) language=self.original_language,
initial_prompt=init_prompt,
verbose=None,
condition_on_previous_text=True,
**self.transcribe_kargs,
)
return result return result
def ts_words(self,r): def ts_words(self, r):
# return: transcribe result object to [(beg,end,"word1"), ...] # return: transcribe result object to [(beg,end,"word1"), ...]
o = [] o = []
for s in r["segments"]: for s in r["segments"]:
for w in s["words"]: for w in s["words"]:
t = (w["start"],w["end"],w["text"]) t = (w["start"], w["end"], w["text"])
o.append(t) o.append(t)
return o return o
@ -95,43 +105,55 @@ class WhisperTimestampedASR(ASRBase):
self.transcribe_kargs["task"] = "translate" self.transcribe_kargs["task"] = "translate"
class FasterWhisperASR(ASRBase): class FasterWhisperASR(ASRBase):
"""Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version. """Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version."""
"""
sep = "" sep = ""
def load_model(self, modelsize=None, cache_dir=None, model_dir=None): def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
from faster_whisper import WhisperModel from faster_whisper import WhisperModel
# logging.getLogger("faster_whisper").setLevel(logger.level)
# logging.getLogger("faster_whisper").setLevel(logger.level)
if model_dir is not None: if model_dir is not None:
logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used.") logger.debug(
f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used."
)
model_size_or_path = model_dir model_size_or_path = model_dir
elif modelsize is not None: elif modelsize is not None:
model_size_or_path = modelsize model_size_or_path = modelsize
else: else:
raise ValueError("modelsize or model_dir parameter must be set") raise ValueError("modelsize or model_dir parameter must be set")
# this worked fast and reliably on NVIDIA L40 # this worked fast and reliably on NVIDIA L40
model = WhisperModel(model_size_or_path, device="cuda", compute_type="float16", download_root=cache_dir) model = WhisperModel(
model_size_or_path,
device="cuda",
compute_type="float16",
download_root=cache_dir,
)
# or run on GPU with INT8 # or run on GPU with INT8
# tested: the transcripts were different, probably worse than with FP16, and it was slightly (appx 20%) slower # tested: the transcripts were different, probably worse than with FP16, and it was slightly (appx 20%) slower
#model = WhisperModel(model_size, device="cuda", compute_type="int8_float16") # model = WhisperModel(model_size, device="cuda", compute_type="int8_float16")
# or run on CPU with INT8 # or run on CPU with INT8
# tested: works, but slow, appx 10-times than cuda FP16 # tested: works, but slow, appx 10-times than cuda FP16
# model = WhisperModel(modelsize, device="cpu", compute_type="int8") #, download_root="faster-disk-cache-dir/") # model = WhisperModel(modelsize, device="cpu", compute_type="int8") #, download_root="faster-disk-cache-dir/")
return model return model
def transcribe(self, audio, init_prompt=""): def transcribe(self, audio, init_prompt=""):
# tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01) # tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01)
segments, info = self.model.transcribe(audio, language=self.original_language, initial_prompt=init_prompt, beam_size=5, word_timestamps=True, condition_on_previous_text=True, **self.transcribe_kargs) segments, info = self.model.transcribe(
#print(info) # info contains language detection result audio,
language=self.original_language,
initial_prompt=init_prompt,
beam_size=5,
word_timestamps=True,
condition_on_previous_text=True,
**self.transcribe_kargs,
)
# print(info) # info contains language detection result
return list(segments) return list(segments)
@ -156,6 +178,7 @@ class FasterWhisperASR(ASRBase):
def set_translate_task(self): def set_translate_task(self):
self.transcribe_kargs["task"] = "translate" self.transcribe_kargs["task"] = "translate"
class MLXWhisper(ASRBase): class MLXWhisper(ASRBase):
""" """
Uses MPX Whisper library as the backend, optimized for Apple Silicon. Uses MPX Whisper library as the backend, optimized for Apple Silicon.
@ -181,11 +204,15 @@ class MLXWhisper(ASRBase):
from mlx_whisper import transcribe from mlx_whisper import transcribe
if model_dir is not None: if model_dir is not None:
logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used.") logger.debug(
f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used."
)
model_size_or_path = model_dir model_size_or_path = model_dir
elif modelsize is not None: elif modelsize is not None:
model_size_or_path = self.translate_model_name(modelsize) model_size_or_path = self.translate_model_name(modelsize)
logger.debug(f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used.") logger.debug(
f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used."
)
self.model_size_or_path = model_size_or_path self.model_size_or_path = model_size_or_path
return transcribe return transcribe
@ -214,7 +241,7 @@ class MLXWhisper(ASRBase):
"large-v2": "mlx-community/whisper-large-v2-mlx", "large-v2": "mlx-community/whisper-large-v2-mlx",
"large-v3": "mlx-community/whisper-large-v3-mlx", "large-v3": "mlx-community/whisper-large-v3-mlx",
"large-v3-turbo": "mlx-community/whisper-large-v3-turbo", "large-v3-turbo": "mlx-community/whisper-large-v3-turbo",
"large": "mlx-community/whisper-large-mlx" "large": "mlx-community/whisper-large-mlx",
} }
# Retrieve the corresponding MLX model path # Retrieve the corresponding MLX model path
@ -223,7 +250,9 @@ class MLXWhisper(ASRBase):
if mlx_model_path: if mlx_model_path:
return mlx_model_path return mlx_model_path
else: else:
raise ValueError(f"Model name '{model_name}' is not recognized or not supported.") raise ValueError(
f"Model name '{model_name}' is not recognized or not supported."
)
def transcribe(self, audio, init_prompt=""): def transcribe(self, audio, init_prompt=""):
segments = self.model( segments = self.model(
@ -233,11 +262,10 @@ class MLXWhisper(ASRBase):
word_timestamps=True, word_timestamps=True,
condition_on_previous_text=True, condition_on_previous_text=True,
path_or_hf_repo=self.model_size_or_path, path_or_hf_repo=self.model_size_or_path,
**self.transcribe_kargs **self.transcribe_kargs,
) )
return segments.get("segments", []) return segments.get("segments", [])
def ts_words(self, segments): def ts_words(self, segments):
""" """
Extract timestamped words from transcription segments and skips words with high no-speech probability. Extract timestamped words from transcription segments and skips words with high no-speech probability.
@ -250,7 +278,7 @@ class MLXWhisper(ASRBase):
] ]
def segments_end_ts(self, res): def segments_end_ts(self, res):
return [s['end'] for s in res] return [s["end"] for s in res]
def use_vad(self): def use_vad(self):
self.transcribe_kargs["vad_filter"] = True self.transcribe_kargs["vad_filter"] = True
@ -258,6 +286,7 @@ class MLXWhisper(ASRBase):
def set_translate_task(self): def set_translate_task(self):
self.transcribe_kargs["task"] = "translate" self.transcribe_kargs["task"] = "translate"
class OpenaiApiASR(ASRBase): class OpenaiApiASR(ASRBase):
"""Uses OpenAI's Whisper API for audio transcription.""" """Uses OpenAI's Whisper API for audio transcription."""
@ -265,7 +294,9 @@ class OpenaiApiASR(ASRBase):
self.logfile = logfile self.logfile = logfile
self.modelname = "whisper-1" self.modelname = "whisper-1"
self.original_language = None if lan == "auto" else lan # ISO-639-1 language code self.original_language = (
None if lan == "auto" else lan
) # ISO-639-1 language code
self.response_format = "verbose_json" self.response_format = "verbose_json"
self.temperature = temperature self.temperature = temperature
@ -278,10 +309,12 @@ class OpenaiApiASR(ASRBase):
def load_model(self, *args, **kwargs): def load_model(self, *args, **kwargs):
from openai import OpenAI from openai import OpenAI
self.client = OpenAI() self.client = OpenAI()
self.transcribed_seconds = 0 # for logging how many seconds were processed by API, to know the cost self.transcribed_seconds = (
0 # for logging how many seconds were processed by API, to know the cost
)
def ts_words(self, segments): def ts_words(self, segments):
no_speech_segments = [] no_speech_segments = []
@ -289,7 +322,9 @@ class OpenaiApiASR(ASRBase):
for segment in segments.segments: for segment in segments.segments:
# TODO: threshold can be set from outside # TODO: threshold can be set from outside
if segment["no_speech_prob"] > 0.8: if segment["no_speech_prob"] > 0.8:
no_speech_segments.append((segment.get("start"), segment.get("end"))) no_speech_segments.append(
(segment.get("start"), segment.get("end"))
)
o = [] o = []
for word in segments.words: for word in segments.words:
@ -301,7 +336,6 @@ class OpenaiApiASR(ASRBase):
o.append((start, end, word.word)) o.append((start, end, word.word))
return o return o
def segments_end_ts(self, res): def segments_end_ts(self, res):
return [s.end for s in res.words] return [s.end for s in res.words]
@ -309,17 +343,19 @@ class OpenaiApiASR(ASRBase):
# Write the audio data to a buffer # Write the audio data to a buffer
buffer = io.BytesIO() buffer = io.BytesIO()
buffer.name = "temp.wav" buffer.name = "temp.wav"
sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16') sf.write(buffer, audio_data, samplerate=16000, format="WAV", subtype="PCM_16")
buffer.seek(0) # Reset buffer's position to the beginning buffer.seek(0) # Reset buffer's position to the beginning
self.transcribed_seconds += math.ceil(len(audio_data)/16000) # it rounds up to the whole seconds self.transcribed_seconds += math.ceil(
len(audio_data) / 16000
) # it rounds up to the whole seconds
params = { params = {
"model": self.modelname, "model": self.modelname,
"file": buffer, "file": buffer,
"response_format": self.response_format, "response_format": self.response_format,
"temperature": self.temperature, "temperature": self.temperature,
"timestamp_granularities": ["word", "segment"] "timestamp_granularities": ["word", "segment"],
} }
if self.task != "translate" and self.original_language: if self.task != "translate" and self.original_language:
params["language"] = self.original_language params["language"] = self.original_language
@ -333,7 +369,9 @@ class OpenaiApiASR(ASRBase):
# Process transcription/translation # Process transcription/translation
transcript = proc.create(**params) transcript = proc.create(**params)
logger.debug(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds") logger.debug(
f"OpenAI API processed accumulated {self.transcribed_seconds} seconds"
)
return transcript return transcript
@ -344,8 +382,6 @@ class OpenaiApiASR(ASRBase):
self.task = "translate" self.task = "translate"
class HypothesisBuffer: class HypothesisBuffer:
def __init__(self, logfile=sys.stderr): def __init__(self, logfile=sys.stderr):
@ -362,19 +398,23 @@ class HypothesisBuffer:
# compare self.commited_in_buffer and new. It inserts only the words in new that extend the commited_in_buffer, it means they are roughly behind last_commited_time and new in content # compare self.commited_in_buffer and new. It inserts only the words in new that extend the commited_in_buffer, it means they are roughly behind last_commited_time and new in content
# the new tail is added to self.new # the new tail is added to self.new
new = [(a+offset,b+offset,t) for a,b,t in new] new = [(a + offset, b + offset, t) for a, b, t in new]
self.new = [(a,b,t) for a,b,t in new if a > self.last_commited_time-0.1] self.new = [(a, b, t) for a, b, t in new if a > self.last_commited_time - 0.1]
if len(self.new) >= 1: if len(self.new) >= 1:
a,b,t = self.new[0] a, b, t = self.new[0]
if abs(a - self.last_commited_time) < 1: if abs(a - self.last_commited_time) < 1:
if self.commited_in_buffer: if self.commited_in_buffer:
# it's going to search for 1, 2, ..., 5 consecutive words (n-grams) that are identical in commited and new. If they are, they're dropped. # it's going to search for 1, 2, ..., 5 consecutive words (n-grams) that are identical in commited and new. If they are, they're dropped.
cn = len(self.commited_in_buffer) cn = len(self.commited_in_buffer)
nn = len(self.new) nn = len(self.new)
for i in range(1,min(min(cn,nn),5)+1): # 5 is the maximum for i in range(1, min(min(cn, nn), 5) + 1): # 5 is the maximum
c = " ".join([self.commited_in_buffer[-j][2] for j in range(1,i+1)][::-1]) c = " ".join(
tail = " ".join(self.new[j-1][2] for j in range(1,i+1)) [self.commited_in_buffer[-j][2] for j in range(1, i + 1)][
::-1
]
)
tail = " ".join(self.new[j - 1][2] for j in range(1, i + 1))
if c == tail: if c == tail:
words = [] words = []
for j in range(i): for j in range(i):
@ -394,7 +434,7 @@ class HypothesisBuffer:
break break
if nt == self.buffer[0][2]: if nt == self.buffer[0][2]:
commit.append((na,nb,nt)) commit.append((na, nb, nt))
self.last_commited_word = nt self.last_commited_word = nt
self.last_commited_time = nb self.last_commited_time = nb
self.buffer.pop(0) self.buffer.pop(0)
@ -413,11 +453,14 @@ class HypothesisBuffer:
def complete(self): def complete(self):
return self.buffer return self.buffer
class OnlineASRProcessor: class OnlineASRProcessor:
SAMPLING_RATE = 16000 SAMPLING_RATE = 16000
def __init__(self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr): def __init__(
self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr
):
"""asr: WhisperASR object """asr: WhisperASR object
tokenizer: sentence tokenizer object for the target language. Must have a method *split* that behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all. tokenizer: sentence tokenizer object for the target language. Must have a method *split* that behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all.
("segment", 15) ("segment", 15)
@ -434,7 +477,7 @@ class OnlineASRProcessor:
def init(self, offset=None): def init(self, offset=None):
"""run this when starting or restarting processing""" """run this when starting or restarting processing"""
self.audio_buffer = np.array([],dtype=np.float32) self.audio_buffer = np.array([], dtype=np.float32)
self.transcript_buffer = HypothesisBuffer(logfile=self.logfile) self.transcript_buffer = HypothesisBuffer(logfile=self.logfile)
self.buffer_time_offset = 0 self.buffer_time_offset = 0
if offset is not None: if offset is not None:
@ -449,20 +492,22 @@ class OnlineASRProcessor:
"""Returns a tuple: (prompt, context), where "prompt" is a 200-character suffix of commited text that is inside of the scrolled away part of audio buffer. """Returns a tuple: (prompt, context), where "prompt" is a 200-character suffix of commited text that is inside of the scrolled away part of audio buffer.
"context" is the commited text that is inside the audio buffer. It is transcribed again and skipped. It is returned only for debugging and logging reasons. "context" is the commited text that is inside the audio buffer. It is transcribed again and skipped. It is returned only for debugging and logging reasons.
""" """
k = max(0,len(self.commited)-1) k = max(0, len(self.commited) - 1)
while k > 0 and self.commited[k-1][1] > self.buffer_time_offset: while k > 0 and self.commited[k - 1][1] > self.buffer_time_offset:
k -= 1 k -= 1
p = self.commited[:k] p = self.commited[:k]
p = [t for _,_,t in p] p = [t for _, _, t in p]
prompt = [] prompt = []
l = 0 l = 0
while p and l < 200: # 200 characters prompt size while p and l < 200: # 200 characters prompt size
x = p.pop(-1) x = p.pop(-1)
l += len(x)+1 l += len(x) + 1
prompt.append(x) prompt.append(x)
non_prompt = self.commited[k:] non_prompt = self.commited[k:]
return self.asr.sep.join(prompt[::-1]), self.asr.sep.join(t for _,_,t in non_prompt) return self.asr.sep.join(prompt[::-1]), self.asr.sep.join(
t for _, _, t in non_prompt
)
def process_iter(self): def process_iter(self):
"""Runs on the current audio buffer. """Runs on the current audio buffer.
@ -473,7 +518,9 @@ class OnlineASRProcessor:
prompt, non_prompt = self.prompt() prompt, non_prompt = self.prompt()
logger.debug(f"PROMPT: {prompt}") logger.debug(f"PROMPT: {prompt}")
logger.debug(f"CONTEXT: {non_prompt}") logger.debug(f"CONTEXT: {non_prompt}")
logger.debug(f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}") logger.debug(
f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}"
)
res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt) res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt)
# transform to [(beg,end,"word1"), ...] # transform to [(beg,end,"word1"), ...]
@ -490,33 +537,37 @@ class OnlineASRProcessor:
# there is a newly confirmed text # there is a newly confirmed text
if o and self.buffer_trimming_way == "sentence": # trim the completed sentences if o and self.buffer_trimming_way == "sentence": # trim the completed sentences
if len(self.audio_buffer)/self.SAMPLING_RATE > self.buffer_trimming_sec: # longer than this if (
len(self.audio_buffer) / self.SAMPLING_RATE > self.buffer_trimming_sec
): # longer than this
self.chunk_completed_sentence() self.chunk_completed_sentence()
if self.buffer_trimming_way == "segment": if self.buffer_trimming_way == "segment":
s = self.buffer_trimming_sec # trim the completed segments longer than s, s = self.buffer_trimming_sec # trim the completed segments longer than s,
else: else:
s = 30 # if the audio buffer is longer than 30s, trim it s = 30 # if the audio buffer is longer than 30s, trim it
if len(self.audio_buffer)/self.SAMPLING_RATE > s: if len(self.audio_buffer) / self.SAMPLING_RATE > s:
self.chunk_completed_segment(res) self.chunk_completed_segment(res)
# alternative: on any word # alternative: on any word
#l = self.buffer_time_offset + len(self.audio_buffer)/self.SAMPLING_RATE - 10 # l = self.buffer_time_offset + len(self.audio_buffer)/self.SAMPLING_RATE - 10
# let's find commited word that is less # let's find commited word that is less
#k = len(self.commited)-1 # k = len(self.commited)-1
#while k>0 and self.commited[k][1] > l: # while k>0 and self.commited[k][1] > l:
# k -= 1 # k -= 1
#t = self.commited[k][1] # t = self.commited[k][1]
logger.debug("chunking segment") logger.debug("chunking segment")
#self.chunk_at(t) # self.chunk_at(t)
logger.debug(f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}") logger.debug(
f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}"
)
return self.to_flush(o) return self.to_flush(o)
def chunk_completed_sentence(self): def chunk_completed_sentence(self):
if self.commited == []: return if self.commited == []:
return
logger.debug(self.commited) logger.debug(self.commited)
sents = self.words_to_sentences(self.commited) sents = self.words_to_sentences(self.commited)
for s in sents: for s in sents:
@ -532,7 +583,8 @@ class OnlineASRProcessor:
self.chunk_at(chunk_at) self.chunk_at(chunk_at)
def chunk_completed_segment(self, res): def chunk_completed_segment(self, res):
if self.commited == []: return if self.commited == []:
return
ends = self.asr.segments_end_ts(res) ends = self.asr.segments_end_ts(res)
@ -540,10 +592,10 @@ class OnlineASRProcessor:
if len(ends) > 1: if len(ends) > 1:
e = ends[-2]+self.buffer_time_offset e = ends[-2] + self.buffer_time_offset
while len(ends) > 2 and e > t: while len(ends) > 2 and e > t:
ends.pop(-1) ends.pop(-1)
e = ends[-2]+self.buffer_time_offset e = ends[-2] + self.buffer_time_offset
if e <= t: if e <= t:
logger.debug(f"--- segment chunked at {e:2.2f}") logger.debug(f"--- segment chunked at {e:2.2f}")
self.chunk_at(e) self.chunk_at(e)
@ -552,16 +604,11 @@ class OnlineASRProcessor:
else: else:
logger.debug(f"--- not enough segments to chunk") logger.debug(f"--- not enough segments to chunk")
def chunk_at(self, time): def chunk_at(self, time):
"""trims the hypothesis and audio buffer at "time" """trims the hypothesis and audio buffer at "time" """
"""
self.transcript_buffer.pop_commited(time) self.transcript_buffer.pop_commited(time)
cut_seconds = time - self.buffer_time_offset cut_seconds = time - self.buffer_time_offset
self.audio_buffer = self.audio_buffer[int(cut_seconds*self.SAMPLING_RATE):] self.audio_buffer = self.audio_buffer[int(cut_seconds * self.SAMPLING_RATE) :]
self.buffer_time_offset = time self.buffer_time_offset = time
def words_to_sentences(self, words): def words_to_sentences(self, words):
@ -579,15 +626,15 @@ class OnlineASRProcessor:
sent = s.pop(0).strip() sent = s.pop(0).strip()
fsent = sent fsent = sent
while cwords: while cwords:
b,e,w = cwords.pop(0) b, e, w = cwords.pop(0)
w = w.strip() w = w.strip()
if beg is None and sent.startswith(w): if beg is None and sent.startswith(w):
beg = b beg = b
elif end is None and sent == w: elif end is None and sent == w:
end = e end = e
out.append((beg,end,fsent)) out.append((beg, end, fsent))
break break
sent = sent[len(w):].strip() sent = sent[len(w) :].strip()
return out return out
def finish(self): def finish(self):
@ -597,11 +644,15 @@ class OnlineASRProcessor:
o = self.transcript_buffer.complete() o = self.transcript_buffer.complete()
f = self.to_flush(o) f = self.to_flush(o)
logger.debug(f"last, noncommited: {f}") logger.debug(f"last, noncommited: {f}")
self.buffer_time_offset += len(self.audio_buffer)/16000 self.buffer_time_offset += len(self.audio_buffer) / 16000
return f return f
def to_flush(
def to_flush(self, sents, sep=None, offset=0, ): self,
sents,
sep=None,
offset=0,
):
# concatenates the timestamped words or sentences into one sequence that is flushed in one line # concatenates the timestamped words or sentences into one sequence that is flushed in one line
# sents: [(beg1, end1, "sentence1"), ...] or [] if empty # sents: [(beg1, end1, "sentence1"), ...] or [] if empty
# return: (beg1,end-of-last-sentence,"concatenation of sentences") or (None, None, "") if empty # return: (beg1,end-of-last-sentence,"concatenation of sentences") or (None, None, "") if empty
@ -614,15 +665,16 @@ class OnlineASRProcessor:
else: else:
b = offset + sents[0][0] b = offset + sents[0][0]
e = offset + sents[-1][1] e = offset + sents[-1][1]
return (b,e,t) return (b, e, t)
class VACOnlineASRProcessor(OnlineASRProcessor): class VACOnlineASRProcessor(OnlineASRProcessor):
'''Wraps OnlineASRProcessor with VAC (Voice Activity Controller). """Wraps OnlineASRProcessor with VAC (Voice Activity Controller).
It works the same way as OnlineASRProcessor: it receives chunks of audio (e.g. 0.04 seconds), It works the same way as OnlineASRProcessor: it receives chunks of audio (e.g. 0.04 seconds),
it runs VAD and continuously detects whether there is speech or not. it runs VAD and continuously detects whether there is speech or not.
When it detects end of speech (non-voice for 500ms), it makes OnlineASRProcessor to end the utterance immediately. When it detects end of speech (non-voice for 500ms), it makes OnlineASRProcessor to end the utterance immediately.
''' """
def __init__(self, online_chunk_size, *a, **kw): def __init__(self, online_chunk_size, *a, **kw):
self.online_chunk_size = online_chunk_size self.online_chunk_size = online_chunk_size
@ -631,12 +683,13 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
# VAC: # VAC:
import torch import torch
model, _ = torch.hub.load(
repo_or_dir='snakers4/silero-vad', model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")
model='silero_vad'
)
from silero_vad_iterator import FixedVADIterator from silero_vad_iterator import FixedVADIterator
self.vac = FixedVADIterator(model) # we use the default options there: 500ms silence, 100ms padding, etc.
self.vac = FixedVADIterator(
model
) # we use the default options there: 500ms silence, 100ms padding, etc.
self.logfile = self.online.logfile self.logfile = self.online.logfile
self.init() self.init()
@ -649,60 +702,65 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
self.is_currently_final = False self.is_currently_final = False
self.status = None # or "voice" or "nonvoice" self.status = None # or "voice" or "nonvoice"
self.audio_buffer = np.array([],dtype=np.float32) self.audio_buffer = np.array([], dtype=np.float32)
self.buffer_offset = 0 # in frames self.buffer_offset = 0 # in frames
def clear_buffer(self): def clear_buffer(self):
self.buffer_offset += len(self.audio_buffer) self.buffer_offset += len(self.audio_buffer)
self.audio_buffer = np.array([],dtype=np.float32) self.audio_buffer = np.array([], dtype=np.float32)
def insert_audio_chunk(self, audio): def insert_audio_chunk(self, audio):
res = self.vac(audio) res = self.vac(audio)
self.audio_buffer = np.append(self.audio_buffer, audio) self.audio_buffer = np.append(self.audio_buffer, audio)
if res is not None: if res is not None:
frame = list(res.values())[0]-self.buffer_offset frame = list(res.values())[0] - self.buffer_offset
if 'start' in res and 'end' not in res: if "start" in res and "end" not in res:
self.status = 'voice' self.status = "voice"
send_audio = self.audio_buffer[frame:] send_audio = self.audio_buffer[frame:]
self.online.init(offset=(frame+self.buffer_offset)/self.SAMPLING_RATE) self.online.init(
offset=(frame + self.buffer_offset) / self.SAMPLING_RATE
)
self.online.insert_audio_chunk(send_audio) self.online.insert_audio_chunk(send_audio)
self.current_online_chunk_buffer_size += len(send_audio) self.current_online_chunk_buffer_size += len(send_audio)
self.clear_buffer() self.clear_buffer()
elif 'end' in res and 'start' not in res: elif "end" in res and "start" not in res:
self.status = 'nonvoice' self.status = "nonvoice"
send_audio = self.audio_buffer[:frame] send_audio = self.audio_buffer[:frame]
self.online.insert_audio_chunk(send_audio) self.online.insert_audio_chunk(send_audio)
self.current_online_chunk_buffer_size += len(send_audio) self.current_online_chunk_buffer_size += len(send_audio)
self.is_currently_final = True self.is_currently_final = True
self.clear_buffer() self.clear_buffer()
else: else:
beg = res["start"]-self.buffer_offset beg = res["start"] - self.buffer_offset
end = res["end"]-self.buffer_offset end = res["end"] - self.buffer_offset
self.status = 'nonvoice' self.status = "nonvoice"
send_audio = self.audio_buffer[beg:end] send_audio = self.audio_buffer[beg:end]
self.online.init(offset=(beg+self.buffer_offset)/self.SAMPLING_RATE) self.online.init(offset=(beg + self.buffer_offset) / self.SAMPLING_RATE)
self.online.insert_audio_chunk(send_audio) self.online.insert_audio_chunk(send_audio)
self.current_online_chunk_buffer_size += len(send_audio) self.current_online_chunk_buffer_size += len(send_audio)
self.is_currently_final = True self.is_currently_final = True
self.clear_buffer() self.clear_buffer()
else: else:
if self.status == 'voice': if self.status == "voice":
self.online.insert_audio_chunk(self.audio_buffer) self.online.insert_audio_chunk(self.audio_buffer)
self.current_online_chunk_buffer_size += len(self.audio_buffer) self.current_online_chunk_buffer_size += len(self.audio_buffer)
self.clear_buffer() self.clear_buffer()
else: else:
# We keep 1 second because VAD may later find start of voice in it. # We keep 1 second because VAD may later find start of voice in it.
# But we trim it to prevent OOM. # But we trim it to prevent OOM.
self.buffer_offset += max(0,len(self.audio_buffer)-self.SAMPLING_RATE) self.buffer_offset += max(
self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE:] 0, len(self.audio_buffer) - self.SAMPLING_RATE
)
self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE :]
def process_iter(self): def process_iter(self):
if self.is_currently_final: if self.is_currently_final:
return self.finish() return self.finish()
elif self.current_online_chunk_buffer_size > self.SAMPLING_RATE*self.online_chunk_size: elif (
self.current_online_chunk_buffer_size
> self.SAMPLING_RATE * self.online_chunk_size
):
self.current_online_chunk_buffer_size = 0 self.current_online_chunk_buffer_size = 0
ret = self.online.process_iter() ret = self.online.process_iter()
return ret return ret
@ -717,37 +775,55 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
return ret return ret
WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(
","
)
WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(",")
def create_tokenizer(lan): def create_tokenizer(lan):
"""returns an object that has split function that works like the one of MosesTokenizer""" """returns an object that has split function that works like the one of MosesTokenizer"""
assert lan in WHISPER_LANG_CODES, "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES) assert (
lan in WHISPER_LANG_CODES
), "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
if lan == "uk": if lan == "uk":
import tokenize_uk import tokenize_uk
class UkrainianTokenizer: class UkrainianTokenizer:
def split(self, text): def split(self, text):
return tokenize_uk.tokenize_sents(text) return tokenize_uk.tokenize_sents(text)
return UkrainianTokenizer() return UkrainianTokenizer()
# supported by fast-mosestokenizer # supported by fast-mosestokenizer
if lan in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split(): if (
lan
in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split()
):
from mosestokenizer import MosesTokenizer from mosestokenizer import MosesTokenizer
return MosesTokenizer(lan) return MosesTokenizer(lan)
# the following languages are in Whisper, but not in wtpsplit: # the following languages are in Whisper, but not in wtpsplit:
if lan in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split(): if (
logger.debug(f"{lan} code is not supported by wtpsplit. Going to use None lang_code option.") lan
in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split()
):
logger.debug(
f"{lan} code is not supported by wtpsplit. Going to use None lang_code option."
)
lan = None lan = None
from wtpsplit import WtP from wtpsplit import WtP
# downloads the model from huggingface on the first use # downloads the model from huggingface on the first use
wtp = WtP("wtp-canine-s-12l-no-adapters") wtp = WtP("wtp-canine-s-12l-no-adapters")
class WtPtok: class WtPtok:
def split(self, sent): def split(self, sent):
return wtp.split(sent, lang_code=lan) return wtp.split(sent, lang_code=lan)
return WtPtok() return WtPtok()
@ -755,19 +831,91 @@ def add_shared_args(parser):
"""shared args for simulation (this entry point) and server """shared args for simulation (this entry point) and server
parser: argparse.ArgumentParser object parser: argparse.ArgumentParser object
""" """
parser.add_argument('--min-chunk-size', type=float, default=1.0, help='Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.') parser.add_argument(
parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.") "--min-chunk-size",
parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved") type=float,
parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.") default=1.0,
parser.add_argument('--lan', '--language', type=str, default='auto', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.") help="Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.",
parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.") )
parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"],help='Load only this backend for Whisper processing.') parser.add_argument(
parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller. Recommended. Requires torch.') "--model",
parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.') type=str,
parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.') default="large-v2",
parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.') choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo".split(
parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.') ","
parser.add_argument("-l", "--log-level", dest="log_level", choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help="Set the log level", default='DEBUG') ),
help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.",
)
parser.add_argument(
"--model_cache_dir",
type=str,
default=None,
help="Overriding the default model cache dir where models downloaded from the hub are saved",
)
parser.add_argument(
"--model_dir",
type=str,
default=None,
help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.",
)
parser.add_argument(
"--lan",
"--language",
type=str,
default="auto",
help="Source language code, e.g. en,de,cs, or 'auto' for language detection.",
)
parser.add_argument(
"--task",
type=str,
default="transcribe",
choices=["transcribe", "translate"],
help="Transcribe or translate.",
)
parser.add_argument(
"--backend",
type=str,
default="faster-whisper",
choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"],
help="Load only this backend for Whisper processing.",
)
parser.add_argument(
"--vac",
action="store_true",
default=False,
help="Use VAC = voice activity controller. Recommended. Requires torch.",
)
parser.add_argument(
"--vac-chunk-size", type=float, default=0.04, help="VAC sample size in seconds."
)
parser.add_argument(
"--vad",
action="store_true",
default=False,
help="Use VAD = voice activity detection, with the default parameters.",
)
parser.add_argument(
"--buffer_trimming",
type=str,
default="segment",
choices=["sentence", "segment"],
help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.',
)
parser.add_argument(
"--buffer_trimming_sec",
type=float,
default=15,
help="Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.",
)
parser.add_argument(
"-l",
"--log-level",
dest="log_level",
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
help="Set the log level",
default="DEBUG",
)
def asr_factory(args, logfile=sys.stderr): def asr_factory(args, logfile=sys.stderr):
""" """
@ -789,12 +937,17 @@ def asr_factory(args, logfile=sys.stderr):
size = args.model size = args.model
t = time.time() t = time.time()
logger.info(f"Loading Whisper {size} model for {args.lan}...") logger.info(f"Loading Whisper {size} model for {args.lan}...")
asr = asr_cls(modelsize=size, lan=args.lan, cache_dir=args.model_cache_dir, model_dir=args.model_dir) asr = asr_cls(
modelsize=size,
lan=args.lan,
cache_dir=args.model_cache_dir,
model_dir=args.model_dir,
)
e = time.time() e = time.time()
logger.info(f"done. It took {round(e-t,2)} seconds.") logger.info(f"done. It took {round(e-t,2)} seconds.")
# Apply common configurations # Apply common configurations
if getattr(args, 'vad', False): # Checks if VAD argument is present and True if getattr(args, "vad", False): # Checks if VAD argument is present and True
logger.info("Setting VAD filter") logger.info("Setting VAD filter")
asr.use_vad() asr.use_vad()
@ -814,30 +967,59 @@ def asr_factory(args, logfile=sys.stderr):
# Create the OnlineASRProcessor # Create the OnlineASRProcessor
if args.vac: if args.vac:
online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) online = VACOnlineASRProcessor(
args.min_chunk_size,
asr,
tokenizer,
logfile=logfile,
buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec),
)
else: else:
online = OnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec)) online = OnlineASRProcessor(
asr,
tokenizer,
logfile=logfile,
buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec),
)
return asr, online return asr, online
def set_logging(args,logger,other="_server"):
logging.basicConfig(#format='%(name)s
format='%(levelname)s\t%(message)s')
logger.setLevel(args.log_level)
logging.getLogger("whisper_online"+other).setLevel(args.log_level)
# logging.getLogger("whisper_online_server").setLevel(args.log_level)
def set_logging(args, logger, other="_server"):
logging.basicConfig(format="%(levelname)s\t%(message)s") # format='%(name)s
logger.setLevel(args.log_level)
logging.getLogger("whisper_online" + other).setLevel(args.log_level)
# logging.getLogger("whisper_online_server").setLevel(args.log_level)
if __name__ == "__main__": if __name__ == "__main__":
import argparse import argparse
parser = argparse.ArgumentParser() parser = argparse.ArgumentParser()
parser.add_argument('audio_path', type=str, help="Filename of 16kHz mono channel wav, on which live streaming is simulated.") parser.add_argument(
"audio_path",
type=str,
help="Filename of 16kHz mono channel wav, on which live streaming is simulated.",
)
add_shared_args(parser) add_shared_args(parser)
parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.') parser.add_argument(
parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.') "--start_at",
parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.') type=float,
default=0.0,
help="Start processing audio at this time.",
)
parser.add_argument(
"--offline", action="store_true", default=False, help="Offline mode."
)
parser.add_argument(
"--comp_unaware",
action="store_true",
default=False,
help="Computationally unaware simulation.",
)
args = parser.parse_args() args = parser.parse_args()
@ -845,19 +1027,21 @@ if __name__ == "__main__":
logfile = sys.stderr logfile = sys.stderr
if args.offline and args.comp_unaware: if args.offline and args.comp_unaware:
logger.error("No or one option from --offline and --comp_unaware are available, not both. Exiting.") logger.error(
"No or one option from --offline and --comp_unaware are available, not both. Exiting."
)
sys.exit(1) sys.exit(1)
# if args.log_level: # if args.log_level:
# logging.basicConfig(format='whisper-%(levelname)s:%(name)s: %(message)s', # logging.basicConfig(format='whisper-%(levelname)s:%(name)s: %(message)s',
# level=getattr(logging, args.log_level)) # level=getattr(logging, args.log_level))
set_logging(args,logger) set_logging(args, logger)
audio_path = args.audio_path audio_path = args.audio_path
SAMPLING_RATE = 16000 SAMPLING_RATE = 16000
duration = len(load_audio(audio_path))/SAMPLING_RATE duration = len(load_audio(audio_path)) / SAMPLING_RATE
logger.info("Audio duration is: %2.2f seconds" % duration) logger.info("Audio duration is: %2.2f seconds" % duration)
asr, online = asr_factory(args, logfile=logfile) asr, online = asr_factory(args, logfile=logfile)
@ -867,13 +1051,13 @@ if __name__ == "__main__":
min_chunk = args.min_chunk_size min_chunk = args.min_chunk_size
# load the audio into the LRU cache before we start the timer # load the audio into the LRU cache before we start the timer
a = load_audio_chunk(audio_path,0,1) a = load_audio_chunk(audio_path, 0, 1)
# warm up the ASR because the very first transcribe takes much more time than the other # warm up the ASR because the very first transcribe takes much more time than the other
asr.transcribe(a) asr.transcribe(a)
beg = args.start_at beg = args.start_at
start = time.time()-beg start = time.time() - beg
def output_transcript(o, now=None): def output_transcript(o, now=None):
# output format in stdout is like: # output format in stdout is like:
@ -883,10 +1067,17 @@ if __name__ == "__main__":
# - beg and end timestamp of the text segment, as estimated by Whisper model. The timestamps are not accurate, but they're useful anyway # - beg and end timestamp of the text segment, as estimated by Whisper model. The timestamps are not accurate, but they're useful anyway
# - the next words: segment transcript # - the next words: segment transcript
if now is None: if now is None:
now = time.time()-start now = time.time() - start
if o[0] is not None: if o[0] is not None:
print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True) print(
print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True) "%1.4f %1.0f %1.0f %s" % (now * 1000, o[0] * 1000, o[1] * 1000, o[2]),
file=logfile,
flush=True,
)
print(
"%1.4f %1.0f %1.0f %s" % (now * 1000, o[0] * 1000, o[1] * 1000, o[2]),
flush=True,
)
else: else:
# No text, so no output # No text, so no output
pass pass
@ -904,7 +1095,7 @@ if __name__ == "__main__":
elif args.comp_unaware: # computational unaware mode elif args.comp_unaware: # computational unaware mode
end = beg + min_chunk end = beg + min_chunk
while True: while True:
a = load_audio_chunk(audio_path,beg,end) a = load_audio_chunk(audio_path, beg, end)
online.insert_audio_chunk(a) online.insert_audio_chunk(a)
try: try:
o = online.process_iter() o = online.process_iter()
@ -931,10 +1122,10 @@ if __name__ == "__main__":
end = 0 end = 0
while True: while True:
now = time.time() - start now = time.time() - start
if now < end+min_chunk: if now < end + min_chunk:
time.sleep(min_chunk+end-now) time.sleep(min_chunk + end - now)
end = time.time() - start end = time.time() - start
a = load_audio_chunk(audio_path,beg,end) a = load_audio_chunk(audio_path, beg, end)
beg = end beg = end
online.insert_audio_chunk(a) online.insert_audio_chunk(a)
@ -946,7 +1137,9 @@ if __name__ == "__main__":
else: else:
output_transcript(o) output_transcript(o)
now = time.time() - start now = time.time() - start
logger.debug(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}") logger.debug(
f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}"
)
if end >= duration: if end >= duration:
break break