Merge pull request #10 from SilasK/main
More flexibility by using custom tokenize_method + black
This commit is contained in:
Quentin Fuxa
GitHub
commit
58e48bb717
3 changed files with 503 additions and 251 deletions
|
|
@ -6,15 +6,16 @@ import torch
|
|||
|
||||
# Their licence is MIT, same as ours: https://github.com/snakers4/silero-vad/blob/f6b1294cb27590fb2452899df98fb234dfef1134/LICENSE
|
||||
|
||||
|
||||
class VADIterator:
|
||||
def __init__(self,
|
||||
def __init__(
|
||||
self,
|
||||
model,
|
||||
threshold: float = 0.5,
|
||||
sampling_rate: int = 16000,
|
||||
min_silence_duration_ms: int = 500, # makes sense on one recording that I checked
|
||||
speech_pad_ms: int = 100 # same
|
||||
speech_pad_ms: int = 100, # same
|
||||
):
|
||||
|
||||
"""
|
||||
Class for stream imitation
|
||||
|
||||
|
|
@ -41,7 +42,9 @@ class VADIterator:
|
|||
self.sampling_rate = sampling_rate
|
||||
|
||||
if sampling_rate not in [8000, 16000]:
|
||||
raise ValueError('VADIterator does not support sampling rates other than [8000, 16000]')
|
||||
raise ValueError(
|
||||
"VADIterator does not support sampling rates other than [8000, 16000]"
|
||||
)
|
||||
|
||||
self.min_silence_samples = sampling_rate * min_silence_duration_ms / 1000
|
||||
self.speech_pad_samples = sampling_rate * speech_pad_ms / 1000
|
||||
|
|
@ -80,7 +83,13 @@ class VADIterator:
|
|||
if (speech_prob >= self.threshold) and not self.triggered:
|
||||
self.triggered = True
|
||||
speech_start = self.current_sample - self.speech_pad_samples
|
||||
return {'start': int(speech_start) if not return_seconds else round(speech_start / self.sampling_rate, 1)}
|
||||
return {
|
||||
"start": (
|
||||
int(speech_start)
|
||||
if not return_seconds
|
||||
else round(speech_start / self.sampling_rate, 1)
|
||||
)
|
||||
}
|
||||
|
||||
if (speech_prob < self.threshold - 0.15) and self.triggered:
|
||||
if not self.temp_end:
|
||||
|
|
@ -91,19 +100,28 @@ class VADIterator:
|
|||
speech_end = self.temp_end + self.speech_pad_samples
|
||||
self.temp_end = 0
|
||||
self.triggered = False
|
||||
return {'end': int(speech_end) if not return_seconds else round(speech_end / self.sampling_rate, 1)}
|
||||
return {
|
||||
"end": (
|
||||
int(speech_end)
|
||||
if not return_seconds
|
||||
else round(speech_end / self.sampling_rate, 1)
|
||||
)
|
||||
}
|
||||
|
||||
return None
|
||||
|
||||
|
||||
#######################
|
||||
# because Silero now requires exactly 512-sized audio chunks
|
||||
|
||||
import numpy as np
|
||||
|
||||
|
||||
class FixedVADIterator(VADIterator):
|
||||
'''It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once.
|
||||
"""It fixes VADIterator by allowing to process any audio length, not only exactly 512 frames at once.
|
||||
If audio to be processed at once is long and multiple voiced segments detected,
|
||||
then __call__ returns the start of the first segment, and end (or middle, which means no end) of the last segment.
|
||||
'''
|
||||
"""
|
||||
|
||||
def reset_states(self):
|
||||
super().reset_states()
|
||||
|
|
@ -118,21 +136,20 @@ class FixedVADIterator(VADIterator):
|
|||
if ret is None:
|
||||
ret = r
|
||||
elif r is not None:
|
||||
if 'end' in r:
|
||||
ret['end'] = r['end'] # the latter end
|
||||
if 'start' in r and 'end' in ret: # there is an earlier start.
|
||||
if "end" in r:
|
||||
ret["end"] = r["end"] # the latter end
|
||||
if "start" in r and "end" in ret: # there is an earlier start.
|
||||
# Remove end, merging this segment with the previous one.
|
||||
del ret['end']
|
||||
del ret["end"]
|
||||
return ret if ret != {} else None
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# test/demonstrate the need for FixedVADIterator:
|
||||
|
||||
import torch
|
||||
model, _ = torch.hub.load(
|
||||
repo_or_dir='snakers4/silero-vad',
|
||||
model='silero_vad'
|
||||
)
|
||||
|
||||
model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")
|
||||
vac = FixedVADIterator(model)
|
||||
# vac = VADIterator(model) # the second case crashes with this
|
||||
|
||||
|
|
|
|||
|
|
@ -22,10 +22,21 @@ app.add_middleware(
|
|||
|
||||
|
||||
parser = argparse.ArgumentParser(description="Whisper FastAPI Online Server")
|
||||
parser.add_argument("--host", type=str, default='localhost', help="The host address to bind the server to.")
|
||||
parser.add_argument("--port", type=int, default=8000, help="The port number to bind the server to.")
|
||||
parser.add_argument("--warmup-file", type=str, dest="warmup_file",
|
||||
help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .")
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="localhost",
|
||||
help="The host address to bind the server to.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--port", type=int, default=8000, help="The port number to bind the server to."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--warmup-file",
|
||||
type=str,
|
||||
dest="warmup_file",
|
||||
help="The path to a speech audio wav file to warm up Whisper so that the very first chunk processing is fast. It can be e.g. https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav .",
|
||||
)
|
||||
add_shared_args(parser)
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -35,29 +46,38 @@ asr, online = asr_factory(args)
|
|||
with open("src/live_transcription.html", "r") as f:
|
||||
html = f.read()
|
||||
|
||||
|
||||
@app.get("/")
|
||||
async def get():
|
||||
return HTMLResponse(html)
|
||||
|
||||
|
||||
SAMPLE_RATE = 16000
|
||||
CHANNELS = 1
|
||||
SAMPLES_PER_SEC = SAMPLE_RATE * int(args.min_chunk_size)
|
||||
BYTES_PER_SAMPLE = 2 # s16le = 2 bytes per sample
|
||||
BYTES_PER_SEC = SAMPLES_PER_SEC * BYTES_PER_SAMPLE
|
||||
|
||||
|
||||
async def start_ffmpeg_decoder():
|
||||
"""
|
||||
Start an FFmpeg process in async streaming mode that reads WebM from stdin
|
||||
and outputs raw s16le PCM on stdout. Returns the process object.
|
||||
"""
|
||||
process = (
|
||||
ffmpeg
|
||||
.input('pipe:0', format='webm')
|
||||
.output('pipe:1', format='s16le', acodec='pcm_s16le', ac=CHANNELS, ar=str(SAMPLE_RATE))
|
||||
ffmpeg.input("pipe:0", format="webm")
|
||||
.output(
|
||||
"pipe:1",
|
||||
format="s16le",
|
||||
acodec="pcm_s16le",
|
||||
ac=CHANNELS,
|
||||
ar=str(SAMPLE_RATE),
|
||||
)
|
||||
.run_async(pipe_stdin=True, pipe_stdout=True, pipe_stderr=True)
|
||||
)
|
||||
return process
|
||||
|
||||
|
||||
@app.websocket("/asr")
|
||||
async def websocket_endpoint(websocket: WebSocket):
|
||||
await websocket.accept()
|
||||
|
|
@ -65,6 +85,7 @@ async def websocket_endpoint(websocket: WebSocket):
|
|||
|
||||
ffmpeg_process = await start_ffmpeg_decoder()
|
||||
pcm_buffer = bytearray()
|
||||
|
||||
# Continuously read decoded PCM from ffmpeg stdout in a background task
|
||||
async def ffmpeg_stdout_reader():
|
||||
nonlocal pcm_buffer
|
||||
|
|
@ -75,9 +96,15 @@ async def websocket_endpoint(websocket: WebSocket):
|
|||
try:
|
||||
elapsed_time = int(time() - beg)
|
||||
beg = time()
|
||||
chunk = await loop.run_in_executor(None, ffmpeg_process.stdout.read, 32000*elapsed_time)
|
||||
if not chunk: # The first chunk will be almost empty, FFmpeg is still starting up
|
||||
chunk = await loop.run_in_executor(None, ffmpeg_process.stdout.read, 4096)
|
||||
chunk = await loop.run_in_executor(
|
||||
None, ffmpeg_process.stdout.read, 32000 * elapsed_time
|
||||
)
|
||||
if (
|
||||
not chunk
|
||||
): # The first chunk will be almost empty, FFmpeg is still starting up
|
||||
chunk = await loop.run_in_executor(
|
||||
None, ffmpeg_process.stdout.read, 4096
|
||||
)
|
||||
if not chunk: # FFmpeg might have closed
|
||||
print("FFmpeg stdout closed.")
|
||||
break
|
||||
|
|
@ -86,21 +113,29 @@ async def websocket_endpoint(websocket: WebSocket):
|
|||
|
||||
if len(pcm_buffer) >= BYTES_PER_SEC:
|
||||
# Convert int16 -> float32
|
||||
pcm_array = np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32) / 32768.0
|
||||
pcm_array = (
|
||||
np.frombuffer(pcm_buffer, dtype=np.int16).astype(np.float32)
|
||||
/ 32768.0
|
||||
)
|
||||
pcm_buffer = bytearray()
|
||||
online.insert_audio_chunk(pcm_array)
|
||||
transcription = online.process_iter()[2]
|
||||
full_transcription += transcription
|
||||
if args.vac:
|
||||
buffer = online.online.to_flush(online.online.transcript_buffer.buffer)[2] # We need to access the underlying online object to get the buffer
|
||||
buffer = online.online.to_flush(
|
||||
online.online.transcript_buffer.buffer
|
||||
)[
|
||||
2
|
||||
] # We need to access the underlying online object to get the buffer
|
||||
else:
|
||||
buffer = online.to_flush(online.transcript_buffer.buffer)[2]
|
||||
if buffer in full_transcription: # With VAC, the buffer is not updated until the next chunk is processed
|
||||
if (
|
||||
buffer in full_transcription
|
||||
): # With VAC, the buffer is not updated until the next chunk is processed
|
||||
buffer = ""
|
||||
await websocket.send_json({
|
||||
"transcription": transcription,
|
||||
"buffer": buffer
|
||||
})
|
||||
await websocket.send_json(
|
||||
{"transcription": transcription, "buffer": buffer}
|
||||
)
|
||||
except Exception as e:
|
||||
print(f"Exception in ffmpeg_stdout_reader: {e}")
|
||||
break
|
||||
|
|
@ -139,4 +174,7 @@ async def websocket_endpoint(websocket: WebSocket):
|
|||
|
||||
if __name__ == "__main__":
|
||||
import uvicorn
|
||||
uvicorn.run("whisper_fastapi_online_server:app", host=args.host, port=args.port, reload=True)
|
||||
|
||||
uvicorn.run(
|
||||
"whisper_fastapi_online_server:app", host=args.host, port=args.port, reload=True
|
||||
)
|
||||
|
|
|
|||
|
|
@ -12,11 +12,13 @@ import math
|
|||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@lru_cache(10**6)
|
||||
def load_audio(fname):
|
||||
a, _ = librosa.load(fname, sr=16000, dtype=np.float32)
|
||||
return a
|
||||
|
||||
|
||||
def load_audio_chunk(fname, beg, end):
|
||||
audio = load_audio(fname)
|
||||
beg_s = int(beg * 16000)
|
||||
|
|
@ -26,12 +28,15 @@ def load_audio_chunk(fname, beg, end):
|
|||
|
||||
# Whisper backend
|
||||
|
||||
|
||||
class ASRBase:
|
||||
|
||||
sep = " " # join transcribe words with this character (" " for whisper_timestamped,
|
||||
# "" for faster-whisper because it emits the spaces when neeeded)
|
||||
|
||||
def __init__(self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr):
|
||||
def __init__(
|
||||
self, lan, modelsize=None, cache_dir=None, model_dir=None, logfile=sys.stderr
|
||||
):
|
||||
self.logfile = logfile
|
||||
|
||||
self.transcribe_kargs = {}
|
||||
|
|
@ -42,7 +47,6 @@ class ASRBase:
|
|||
|
||||
self.model = self.load_model(modelsize, cache_dir, model_dir)
|
||||
|
||||
|
||||
def load_model(self, modelsize, cache_dir):
|
||||
raise NotImplemented("must be implemented in the child class")
|
||||
|
||||
|
|
@ -64,16 +68,22 @@ class WhisperTimestampedASR(ASRBase):
|
|||
import whisper
|
||||
import whisper_timestamped
|
||||
from whisper_timestamped import transcribe_timestamped
|
||||
|
||||
self.transcribe_timestamped = transcribe_timestamped
|
||||
if model_dir is not None:
|
||||
logger.debug("ignoring model_dir, not implemented")
|
||||
return whisper.load_model(modelsize, download_root=cache_dir)
|
||||
|
||||
def transcribe(self, audio, init_prompt=""):
|
||||
result = self.transcribe_timestamped(self.model,
|
||||
audio, language=self.original_language,
|
||||
initial_prompt=init_prompt, verbose=None,
|
||||
condition_on_previous_text=True, **self.transcribe_kargs)
|
||||
result = self.transcribe_timestamped(
|
||||
self.model,
|
||||
audio,
|
||||
language=self.original_language,
|
||||
initial_prompt=init_prompt,
|
||||
verbose=None,
|
||||
condition_on_previous_text=True,
|
||||
**self.transcribe_kargs,
|
||||
)
|
||||
return result
|
||||
|
||||
def ts_words(self, r):
|
||||
|
|
@ -95,28 +105,32 @@ class WhisperTimestampedASR(ASRBase):
|
|||
self.transcribe_kargs["task"] = "translate"
|
||||
|
||||
|
||||
|
||||
|
||||
class FasterWhisperASR(ASRBase):
|
||||
"""Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version.
|
||||
"""
|
||||
"""Uses faster-whisper library as the backend. Works much faster, appx 4-times (in offline mode). For GPU, it requires installation with a specific CUDNN version."""
|
||||
|
||||
sep = ""
|
||||
|
||||
def load_model(self, modelsize=None, cache_dir=None, model_dir=None):
|
||||
from faster_whisper import WhisperModel
|
||||
|
||||
# logging.getLogger("faster_whisper").setLevel(logger.level)
|
||||
if model_dir is not None:
|
||||
logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used.")
|
||||
logger.debug(
|
||||
f"Loading whisper model from model_dir {model_dir}. modelsize and cache_dir parameters are not used."
|
||||
)
|
||||
model_size_or_path = model_dir
|
||||
elif modelsize is not None:
|
||||
model_size_or_path = modelsize
|
||||
else:
|
||||
raise ValueError("modelsize or model_dir parameter must be set")
|
||||
|
||||
|
||||
# this worked fast and reliably on NVIDIA L40
|
||||
model = WhisperModel(model_size_or_path, device="cuda", compute_type="float16", download_root=cache_dir)
|
||||
model = WhisperModel(
|
||||
model_size_or_path,
|
||||
device="cuda",
|
||||
compute_type="float16",
|
||||
download_root=cache_dir,
|
||||
)
|
||||
|
||||
# or run on GPU with INT8
|
||||
# tested: the transcripts were different, probably worse than with FP16, and it was slightly (appx 20%) slower
|
||||
|
|
@ -130,7 +144,15 @@ class FasterWhisperASR(ASRBase):
|
|||
def transcribe(self, audio, init_prompt=""):
|
||||
|
||||
# tested: beam_size=5 is faster and better than 1 (on one 200 second document from En ESIC, min chunk 0.01)
|
||||
segments, info = self.model.transcribe(audio, language=self.original_language, initial_prompt=init_prompt, beam_size=5, word_timestamps=True, condition_on_previous_text=True, **self.transcribe_kargs)
|
||||
segments, info = self.model.transcribe(
|
||||
audio,
|
||||
language=self.original_language,
|
||||
initial_prompt=init_prompt,
|
||||
beam_size=5,
|
||||
word_timestamps=True,
|
||||
condition_on_previous_text=True,
|
||||
**self.transcribe_kargs,
|
||||
)
|
||||
# print(info) # info contains language detection result
|
||||
|
||||
return list(segments)
|
||||
|
|
@ -156,6 +178,7 @@ class FasterWhisperASR(ASRBase):
|
|||
def set_translate_task(self):
|
||||
self.transcribe_kargs["task"] = "translate"
|
||||
|
||||
|
||||
class MLXWhisper(ASRBase):
|
||||
"""
|
||||
Uses MPX Whisper library as the backend, optimized for Apple Silicon.
|
||||
|
|
@ -181,11 +204,15 @@ class MLXWhisper(ASRBase):
|
|||
from mlx_whisper import transcribe
|
||||
|
||||
if model_dir is not None:
|
||||
logger.debug(f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used.")
|
||||
logger.debug(
|
||||
f"Loading whisper model from model_dir {model_dir}. modelsize parameter is not used."
|
||||
)
|
||||
model_size_or_path = model_dir
|
||||
elif modelsize is not None:
|
||||
model_size_or_path = self.translate_model_name(modelsize)
|
||||
logger.debug(f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used.")
|
||||
logger.debug(
|
||||
f"Loading whisper model {modelsize}. You use mlx whisper, so {model_size_or_path} will be used."
|
||||
)
|
||||
|
||||
self.model_size_or_path = model_size_or_path
|
||||
return transcribe
|
||||
|
|
@ -214,7 +241,7 @@ class MLXWhisper(ASRBase):
|
|||
"large-v2": "mlx-community/whisper-large-v2-mlx",
|
||||
"large-v3": "mlx-community/whisper-large-v3-mlx",
|
||||
"large-v3-turbo": "mlx-community/whisper-large-v3-turbo",
|
||||
"large": "mlx-community/whisper-large-mlx"
|
||||
"large": "mlx-community/whisper-large-mlx",
|
||||
}
|
||||
|
||||
# Retrieve the corresponding MLX model path
|
||||
|
|
@ -223,7 +250,9 @@ class MLXWhisper(ASRBase):
|
|||
if mlx_model_path:
|
||||
return mlx_model_path
|
||||
else:
|
||||
raise ValueError(f"Model name '{model_name}' is not recognized or not supported.")
|
||||
raise ValueError(
|
||||
f"Model name '{model_name}' is not recognized or not supported."
|
||||
)
|
||||
|
||||
def transcribe(self, audio, init_prompt=""):
|
||||
segments = self.model(
|
||||
|
|
@ -233,11 +262,10 @@ class MLXWhisper(ASRBase):
|
|||
word_timestamps=True,
|
||||
condition_on_previous_text=True,
|
||||
path_or_hf_repo=self.model_size_or_path,
|
||||
**self.transcribe_kargs
|
||||
**self.transcribe_kargs,
|
||||
)
|
||||
return segments.get("segments", [])
|
||||
|
||||
|
||||
def ts_words(self, segments):
|
||||
"""
|
||||
Extract timestamped words from transcription segments and skips words with high no-speech probability.
|
||||
|
|
@ -250,7 +278,7 @@ class MLXWhisper(ASRBase):
|
|||
]
|
||||
|
||||
def segments_end_ts(self, res):
|
||||
return [s['end'] for s in res]
|
||||
return [s["end"] for s in res]
|
||||
|
||||
def use_vad(self):
|
||||
self.transcribe_kargs["vad_filter"] = True
|
||||
|
|
@ -258,6 +286,7 @@ class MLXWhisper(ASRBase):
|
|||
def set_translate_task(self):
|
||||
self.transcribe_kargs["task"] = "translate"
|
||||
|
||||
|
||||
class OpenaiApiASR(ASRBase):
|
||||
"""Uses OpenAI's Whisper API for audio transcription."""
|
||||
|
||||
|
|
@ -265,7 +294,9 @@ class OpenaiApiASR(ASRBase):
|
|||
self.logfile = logfile
|
||||
|
||||
self.modelname = "whisper-1"
|
||||
self.original_language = None if lan == "auto" else lan # ISO-639-1 language code
|
||||
self.original_language = (
|
||||
None if lan == "auto" else lan
|
||||
) # ISO-639-1 language code
|
||||
self.response_format = "verbose_json"
|
||||
self.temperature = temperature
|
||||
|
||||
|
|
@ -278,10 +309,12 @@ class OpenaiApiASR(ASRBase):
|
|||
|
||||
def load_model(self, *args, **kwargs):
|
||||
from openai import OpenAI
|
||||
|
||||
self.client = OpenAI()
|
||||
|
||||
self.transcribed_seconds = 0 # for logging how many seconds were processed by API, to know the cost
|
||||
|
||||
self.transcribed_seconds = (
|
||||
0 # for logging how many seconds were processed by API, to know the cost
|
||||
)
|
||||
|
||||
def ts_words(self, segments):
|
||||
no_speech_segments = []
|
||||
|
|
@ -289,7 +322,9 @@ class OpenaiApiASR(ASRBase):
|
|||
for segment in segments.segments:
|
||||
# TODO: threshold can be set from outside
|
||||
if segment["no_speech_prob"] > 0.8:
|
||||
no_speech_segments.append((segment.get("start"), segment.get("end")))
|
||||
no_speech_segments.append(
|
||||
(segment.get("start"), segment.get("end"))
|
||||
)
|
||||
|
||||
o = []
|
||||
for word in segments.words:
|
||||
|
|
@ -301,7 +336,6 @@ class OpenaiApiASR(ASRBase):
|
|||
o.append((start, end, word.word))
|
||||
return o
|
||||
|
||||
|
||||
def segments_end_ts(self, res):
|
||||
return [s.end for s in res.words]
|
||||
|
||||
|
|
@ -309,17 +343,19 @@ class OpenaiApiASR(ASRBase):
|
|||
# Write the audio data to a buffer
|
||||
buffer = io.BytesIO()
|
||||
buffer.name = "temp.wav"
|
||||
sf.write(buffer, audio_data, samplerate=16000, format='WAV', subtype='PCM_16')
|
||||
sf.write(buffer, audio_data, samplerate=16000, format="WAV", subtype="PCM_16")
|
||||
buffer.seek(0) # Reset buffer's position to the beginning
|
||||
|
||||
self.transcribed_seconds += math.ceil(len(audio_data)/16000) # it rounds up to the whole seconds
|
||||
self.transcribed_seconds += math.ceil(
|
||||
len(audio_data) / 16000
|
||||
) # it rounds up to the whole seconds
|
||||
|
||||
params = {
|
||||
"model": self.modelname,
|
||||
"file": buffer,
|
||||
"response_format": self.response_format,
|
||||
"temperature": self.temperature,
|
||||
"timestamp_granularities": ["word", "segment"]
|
||||
"timestamp_granularities": ["word", "segment"],
|
||||
}
|
||||
if self.task != "translate" and self.original_language:
|
||||
params["language"] = self.original_language
|
||||
|
|
@ -333,7 +369,9 @@ class OpenaiApiASR(ASRBase):
|
|||
|
||||
# Process transcription/translation
|
||||
transcript = proc.create(**params)
|
||||
logger.debug(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds")
|
||||
logger.debug(
|
||||
f"OpenAI API processed accumulated {self.transcribed_seconds} seconds"
|
||||
)
|
||||
|
||||
return transcript
|
||||
|
||||
|
|
@ -344,8 +382,6 @@ class OpenaiApiASR(ASRBase):
|
|||
self.task = "translate"
|
||||
|
||||
|
||||
|
||||
|
||||
class HypothesisBuffer:
|
||||
|
||||
def __init__(self, logfile=sys.stderr):
|
||||
|
|
@ -373,7 +409,11 @@ class HypothesisBuffer:
|
|||
cn = len(self.commited_in_buffer)
|
||||
nn = len(self.new)
|
||||
for i in range(1, min(min(cn, nn), 5) + 1): # 5 is the maximum
|
||||
c = " ".join([self.commited_in_buffer[-j][2] for j in range(1,i+1)][::-1])
|
||||
c = " ".join(
|
||||
[self.commited_in_buffer[-j][2] for j in range(1, i + 1)][
|
||||
::-1
|
||||
]
|
||||
)
|
||||
tail = " ".join(self.new[j - 1][2] for j in range(1, i + 1))
|
||||
if c == tail:
|
||||
words = []
|
||||
|
|
@ -413,19 +453,26 @@ class HypothesisBuffer:
|
|||
def complete(self):
|
||||
return self.buffer
|
||||
|
||||
|
||||
class OnlineASRProcessor:
|
||||
|
||||
SAMPLING_RATE = 16000
|
||||
|
||||
def __init__(self, asr, tokenizer=None, buffer_trimming=("segment", 15), logfile=sys.stderr):
|
||||
def __init__(
|
||||
self,
|
||||
asr,
|
||||
tokenize_method=None,
|
||||
buffer_trimming=("segment", 15),
|
||||
logfile=sys.stderr,
|
||||
):
|
||||
"""asr: WhisperASR object
|
||||
tokenizer: sentence tokenizer object for the target language. Must have a method *split* that behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all.
|
||||
tokenize_method: sentence tokenizer function for the target language. Must be a callable and behaves like the one of MosesTokenizer. It can be None, if "segment" buffer trimming option is used, then tokenizer is not used at all.
|
||||
("segment", 15)
|
||||
buffer_trimming: a pair of (option, seconds), where option is either "sentence" or "segment", and seconds is a number. Buffer is trimmed if it is longer than "seconds" threshold. Default is the most recommended option.
|
||||
logfile: where to store the log.
|
||||
"""
|
||||
self.asr = asr
|
||||
self.tokenizer = tokenizer
|
||||
self.tokenize = tokenize_method
|
||||
self.logfile = logfile
|
||||
|
||||
self.init()
|
||||
|
|
@ -462,7 +509,9 @@ class OnlineASRProcessor:
|
|||
l += len(x) + 1
|
||||
prompt.append(x)
|
||||
non_prompt = self.commited[k:]
|
||||
return self.asr.sep.join(prompt[::-1]), self.asr.sep.join(t for _,_,t in non_prompt)
|
||||
return self.asr.sep.join(prompt[::-1]), self.asr.sep.join(
|
||||
t for _, _, t in non_prompt
|
||||
)
|
||||
|
||||
def process_iter(self):
|
||||
"""Runs on the current audio buffer.
|
||||
|
|
@ -473,7 +522,9 @@ class OnlineASRProcessor:
|
|||
prompt, non_prompt = self.prompt()
|
||||
logger.debug(f"PROMPT: {prompt}")
|
||||
logger.debug(f"CONTEXT: {non_prompt}")
|
||||
logger.debug(f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}")
|
||||
logger.debug(
|
||||
f"transcribing {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f} seconds from {self.buffer_time_offset:2.2f}"
|
||||
)
|
||||
res = self.asr.transcribe(self.audio_buffer, init_prompt=prompt)
|
||||
|
||||
# transform to [(beg,end,"word1"), ...]
|
||||
|
|
@ -483,17 +534,18 @@ class OnlineASRProcessor:
|
|||
o = self.transcript_buffer.flush()
|
||||
self.commited.extend(o)
|
||||
completed = self.to_flush(o)
|
||||
logger.debug(f">>>>COMPLETE NOW: {completed}")
|
||||
logger.debug(f">>>>COMPLETE NOW: {completed[2]}")
|
||||
the_rest = self.to_flush(self.transcript_buffer.complete())
|
||||
logger.debug(f"INCOMPLETE: {the_rest}")
|
||||
logger.debug(f"INCOMPLETE: {the_rest[2]}")
|
||||
|
||||
# there is a newly confirmed text
|
||||
|
||||
if o and self.buffer_trimming_way == "sentence": # trim the completed sentences
|
||||
if len(self.audio_buffer)/self.SAMPLING_RATE > self.buffer_trimming_sec: # longer than this
|
||||
if (
|
||||
len(self.audio_buffer) / self.SAMPLING_RATE > self.buffer_trimming_sec
|
||||
): # longer than this
|
||||
self.chunk_completed_sentence()
|
||||
|
||||
|
||||
if self.buffer_trimming_way == "segment":
|
||||
s = self.buffer_trimming_sec # trim the completed segments longer than s,
|
||||
else:
|
||||
|
|
@ -512,12 +564,15 @@ class OnlineASRProcessor:
|
|||
logger.debug("chunking segment")
|
||||
# self.chunk_at(t)
|
||||
|
||||
logger.debug(f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}")
|
||||
logger.debug(
|
||||
f"len of buffer now: {len(self.audio_buffer)/self.SAMPLING_RATE:2.2f}"
|
||||
)
|
||||
return self.to_flush(o)
|
||||
|
||||
def chunk_completed_sentence(self):
|
||||
if self.commited == []: return
|
||||
logger.debug(self.commited)
|
||||
if self.commited == []:
|
||||
return
|
||||
logger.debug("COMPLETED SENTENCE: ", [s[2] for s in self.commited])
|
||||
sents = self.words_to_sentences(self.commited)
|
||||
for s in sents:
|
||||
logger.debug(f"\t\tSENT: {s}")
|
||||
|
|
@ -532,7 +587,8 @@ class OnlineASRProcessor:
|
|||
self.chunk_at(chunk_at)
|
||||
|
||||
def chunk_completed_segment(self, res):
|
||||
if self.commited == []: return
|
||||
if self.commited == []:
|
||||
return
|
||||
|
||||
ends = self.asr.segments_end_ts(res)
|
||||
|
||||
|
|
@ -552,26 +608,21 @@ class OnlineASRProcessor:
|
|||
else:
|
||||
logger.debug(f"--- not enough segments to chunk")
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def chunk_at(self, time):
|
||||
"""trims the hypothesis and audio buffer at "time"
|
||||
"""
|
||||
"""trims the hypothesis and audio buffer at "time" """
|
||||
self.transcript_buffer.pop_commited(time)
|
||||
cut_seconds = time - self.buffer_time_offset
|
||||
self.audio_buffer = self.audio_buffer[int(cut_seconds * self.SAMPLING_RATE) :]
|
||||
self.buffer_time_offset = time
|
||||
|
||||
def words_to_sentences(self, words):
|
||||
"""Uses self.tokenizer for sentence segmentation of words.
|
||||
"""Uses self.tokenize for sentence segmentation of words.
|
||||
Returns: [(beg,end,"sentence 1"),...]
|
||||
"""
|
||||
|
||||
cwords = [w for w in words]
|
||||
t = " ".join(o[2] for o in cwords)
|
||||
s = self.tokenizer.split(t)
|
||||
s = self.tokenize(t)
|
||||
out = []
|
||||
while s:
|
||||
beg = None
|
||||
|
|
@ -600,8 +651,12 @@ class OnlineASRProcessor:
|
|||
self.buffer_time_offset += len(self.audio_buffer) / 16000
|
||||
return f
|
||||
|
||||
|
||||
def to_flush(self, sents, sep=None, offset=0, ):
|
||||
def to_flush(
|
||||
self,
|
||||
sents,
|
||||
sep=None,
|
||||
offset=0,
|
||||
):
|
||||
# concatenates the timestamped words or sentences into one sequence that is flushed in one line
|
||||
# sents: [(beg1, end1, "sentence1"), ...] or [] if empty
|
||||
# return: (beg1,end-of-last-sentence,"concatenation of sentences") or (None, None, "") if empty
|
||||
|
|
@ -616,13 +671,14 @@ class OnlineASRProcessor:
|
|||
e = offset + sents[-1][1]
|
||||
return (b, e, t)
|
||||
|
||||
|
||||
class VACOnlineASRProcessor(OnlineASRProcessor):
|
||||
'''Wraps OnlineASRProcessor with VAC (Voice Activity Controller).
|
||||
"""Wraps OnlineASRProcessor with VAC (Voice Activity Controller).
|
||||
|
||||
It works the same way as OnlineASRProcessor: it receives chunks of audio (e.g. 0.04 seconds),
|
||||
it runs VAD and continuously detects whether there is speech or not.
|
||||
When it detects end of speech (non-voice for 500ms), it makes OnlineASRProcessor to end the utterance immediately.
|
||||
'''
|
||||
"""
|
||||
|
||||
def __init__(self, online_chunk_size, *a, **kw):
|
||||
self.online_chunk_size = online_chunk_size
|
||||
|
|
@ -631,12 +687,13 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
|
|||
|
||||
# VAC:
|
||||
import torch
|
||||
model, _ = torch.hub.load(
|
||||
repo_or_dir='snakers4/silero-vad',
|
||||
model='silero_vad'
|
||||
)
|
||||
|
||||
model, _ = torch.hub.load(repo_or_dir="snakers4/silero-vad", model="silero_vad")
|
||||
from silero_vad_iterator import FixedVADIterator
|
||||
self.vac = FixedVADIterator(model) # we use the default options there: 500ms silence, 100ms padding, etc.
|
||||
|
||||
self.vac = FixedVADIterator(
|
||||
model
|
||||
) # we use the default options there: 500ms silence, 100ms padding, etc.
|
||||
|
||||
self.logfile = self.online.logfile
|
||||
self.init()
|
||||
|
|
@ -656,22 +713,23 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
|
|||
self.buffer_offset += len(self.audio_buffer)
|
||||
self.audio_buffer = np.array([], dtype=np.float32)
|
||||
|
||||
|
||||
def insert_audio_chunk(self, audio):
|
||||
res = self.vac(audio)
|
||||
self.audio_buffer = np.append(self.audio_buffer, audio)
|
||||
|
||||
if res is not None:
|
||||
frame = list(res.values())[0] - self.buffer_offset
|
||||
if 'start' in res and 'end' not in res:
|
||||
self.status = 'voice'
|
||||
if "start" in res and "end" not in res:
|
||||
self.status = "voice"
|
||||
send_audio = self.audio_buffer[frame:]
|
||||
self.online.init(offset=(frame+self.buffer_offset)/self.SAMPLING_RATE)
|
||||
self.online.init(
|
||||
offset=(frame + self.buffer_offset) / self.SAMPLING_RATE
|
||||
)
|
||||
self.online.insert_audio_chunk(send_audio)
|
||||
self.current_online_chunk_buffer_size += len(send_audio)
|
||||
self.clear_buffer()
|
||||
elif 'end' in res and 'start' not in res:
|
||||
self.status = 'nonvoice'
|
||||
elif "end" in res and "start" not in res:
|
||||
self.status = "nonvoice"
|
||||
send_audio = self.audio_buffer[:frame]
|
||||
self.online.insert_audio_chunk(send_audio)
|
||||
self.current_online_chunk_buffer_size += len(send_audio)
|
||||
|
|
@ -680,7 +738,7 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
|
|||
else:
|
||||
beg = res["start"] - self.buffer_offset
|
||||
end = res["end"] - self.buffer_offset
|
||||
self.status = 'nonvoice'
|
||||
self.status = "nonvoice"
|
||||
send_audio = self.audio_buffer[beg:end]
|
||||
self.online.init(offset=(beg + self.buffer_offset) / self.SAMPLING_RATE)
|
||||
self.online.insert_audio_chunk(send_audio)
|
||||
|
|
@ -688,21 +746,25 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
|
|||
self.is_currently_final = True
|
||||
self.clear_buffer()
|
||||
else:
|
||||
if self.status == 'voice':
|
||||
if self.status == "voice":
|
||||
self.online.insert_audio_chunk(self.audio_buffer)
|
||||
self.current_online_chunk_buffer_size += len(self.audio_buffer)
|
||||
self.clear_buffer()
|
||||
else:
|
||||
# We keep 1 second because VAD may later find start of voice in it.
|
||||
# But we trim it to prevent OOM.
|
||||
self.buffer_offset += max(0,len(self.audio_buffer)-self.SAMPLING_RATE)
|
||||
self.buffer_offset += max(
|
||||
0, len(self.audio_buffer) - self.SAMPLING_RATE
|
||||
)
|
||||
self.audio_buffer = self.audio_buffer[-self.SAMPLING_RATE :]
|
||||
|
||||
|
||||
def process_iter(self):
|
||||
if self.is_currently_final:
|
||||
return self.finish()
|
||||
elif self.current_online_chunk_buffer_size > self.SAMPLING_RATE*self.online_chunk_size:
|
||||
elif (
|
||||
self.current_online_chunk_buffer_size
|
||||
> self.SAMPLING_RATE * self.online_chunk_size
|
||||
):
|
||||
self.current_online_chunk_buffer_size = 0
|
||||
ret = self.online.process_iter()
|
||||
return ret
|
||||
|
|
@ -717,37 +779,55 @@ class VACOnlineASRProcessor(OnlineASRProcessor):
|
|||
return ret
|
||||
|
||||
|
||||
WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(
|
||||
","
|
||||
)
|
||||
|
||||
WHISPER_LANG_CODES = "af,am,ar,as,az,ba,be,bg,bn,bo,br,bs,ca,cs,cy,da,de,el,en,es,et,eu,fa,fi,fo,fr,gl,gu,ha,haw,he,hi,hr,ht,hu,hy,id,is,it,ja,jw,ka,kk,km,kn,ko,la,lb,ln,lo,lt,lv,mg,mi,mk,ml,mn,mr,ms,mt,my,ne,nl,nn,no,oc,pa,pl,ps,pt,ro,ru,sa,sd,si,sk,sl,sn,so,sq,sr,su,sv,sw,ta,te,tg,th,tk,tl,tr,tt,uk,ur,uz,vi,yi,yo,zh".split(",")
|
||||
|
||||
def create_tokenizer(lan):
|
||||
"""returns an object that has split function that works like the one of MosesTokenizer"""
|
||||
|
||||
assert lan in WHISPER_LANG_CODES, "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
|
||||
assert (
|
||||
lan in WHISPER_LANG_CODES
|
||||
), "language must be Whisper's supported lang code: " + " ".join(WHISPER_LANG_CODES)
|
||||
|
||||
if lan == "uk":
|
||||
import tokenize_uk
|
||||
|
||||
class UkrainianTokenizer:
|
||||
def split(self, text):
|
||||
return tokenize_uk.tokenize_sents(text)
|
||||
|
||||
return UkrainianTokenizer()
|
||||
|
||||
# supported by fast-mosestokenizer
|
||||
if lan in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split():
|
||||
if (
|
||||
lan
|
||||
in "as bn ca cs de el en es et fi fr ga gu hi hu is it kn lt lv ml mni mr nl or pa pl pt ro ru sk sl sv ta te yue zh".split()
|
||||
):
|
||||
from mosestokenizer import MosesTokenizer
|
||||
|
||||
return MosesTokenizer(lan)
|
||||
|
||||
# the following languages are in Whisper, but not in wtpsplit:
|
||||
if lan in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split():
|
||||
logger.debug(f"{lan} code is not supported by wtpsplit. Going to use None lang_code option.")
|
||||
if (
|
||||
lan
|
||||
in "as ba bo br bs fo haw hr ht jw lb ln lo mi nn oc sa sd sn so su sw tk tl tt".split()
|
||||
):
|
||||
logger.debug(
|
||||
f"{lan} code is not supported by wtpsplit. Going to use None lang_code option."
|
||||
)
|
||||
lan = None
|
||||
|
||||
from wtpsplit import WtP
|
||||
|
||||
# downloads the model from huggingface on the first use
|
||||
wtp = WtP("wtp-canine-s-12l-no-adapters")
|
||||
|
||||
class WtPtok:
|
||||
def split(self, sent):
|
||||
return wtp.split(sent, lang_code=lan)
|
||||
|
||||
return WtPtok()
|
||||
|
||||
|
||||
|
|
@ -755,19 +835,91 @@ def add_shared_args(parser):
|
|||
"""shared args for simulation (this entry point) and server
|
||||
parser: argparse.ArgumentParser object
|
||||
"""
|
||||
parser.add_argument('--min-chunk-size', type=float, default=1.0, help='Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.')
|
||||
parser.add_argument('--model', type=str, default='large-v2', choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo".split(","),help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.")
|
||||
parser.add_argument('--model_cache_dir', type=str, default=None, help="Overriding the default model cache dir where models downloaded from the hub are saved")
|
||||
parser.add_argument('--model_dir', type=str, default=None, help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.")
|
||||
parser.add_argument('--lan', '--language', type=str, default='auto', help="Source language code, e.g. en,de,cs, or 'auto' for language detection.")
|
||||
parser.add_argument('--task', type=str, default='transcribe', choices=["transcribe","translate"],help="Transcribe or translate.")
|
||||
parser.add_argument('--backend', type=str, default="faster-whisper", choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"],help='Load only this backend for Whisper processing.')
|
||||
parser.add_argument('--vac', action="store_true", default=False, help='Use VAC = voice activity controller. Recommended. Requires torch.')
|
||||
parser.add_argument('--vac-chunk-size', type=float, default=0.04, help='VAC sample size in seconds.')
|
||||
parser.add_argument('--vad', action="store_true", default=False, help='Use VAD = voice activity detection, with the default parameters.')
|
||||
parser.add_argument('--buffer_trimming', type=str, default="segment", choices=["sentence", "segment"],help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.')
|
||||
parser.add_argument('--buffer_trimming_sec', type=float, default=15, help='Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.')
|
||||
parser.add_argument("-l", "--log-level", dest="log_level", choices=['DEBUG', 'INFO', 'WARNING', 'ERROR', 'CRITICAL'], help="Set the log level", default='DEBUG')
|
||||
parser.add_argument(
|
||||
"--min-chunk-size",
|
||||
type=float,
|
||||
default=1.0,
|
||||
help="Minimum audio chunk size in seconds. It waits up to this time to do processing. If the processing takes shorter time, it waits, otherwise it processes the whole segment that was received by this time.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model",
|
||||
type=str,
|
||||
default="large-v2",
|
||||
choices="tiny.en,tiny,base.en,base,small.en,small,medium.en,medium,large-v1,large-v2,large-v3,large,large-v3-turbo".split(
|
||||
","
|
||||
),
|
||||
help="Name size of the Whisper model to use (default: large-v2). The model is automatically downloaded from the model hub if not present in model cache dir.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model_cache_dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Overriding the default model cache dir where models downloaded from the hub are saved",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--model_dir",
|
||||
type=str,
|
||||
default=None,
|
||||
help="Dir where Whisper model.bin and other files are saved. This option overrides --model and --model_cache_dir parameter.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lan",
|
||||
"--language",
|
||||
type=str,
|
||||
default="auto",
|
||||
help="Source language code, e.g. en,de,cs, or 'auto' for language detection.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--task",
|
||||
type=str,
|
||||
default="transcribe",
|
||||
choices=["transcribe", "translate"],
|
||||
help="Transcribe or translate.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--backend",
|
||||
type=str,
|
||||
default="faster-whisper",
|
||||
choices=["faster-whisper", "whisper_timestamped", "mlx-whisper", "openai-api"],
|
||||
help="Load only this backend for Whisper processing.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vac",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Use VAC = voice activity controller. Recommended. Requires torch.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vac-chunk-size", type=float, default=0.04, help="VAC sample size in seconds."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--vad",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Use VAD = voice activity detection, with the default parameters.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buffer_trimming",
|
||||
type=str,
|
||||
default="segment",
|
||||
choices=["sentence", "segment"],
|
||||
help='Buffer trimming strategy -- trim completed sentences marked with punctuation mark and detected by sentence segmenter, or the completed segments returned by Whisper. Sentence segmenter must be installed for "sentence" option.',
|
||||
)
|
||||
parser.add_argument(
|
||||
"--buffer_trimming_sec",
|
||||
type=float,
|
||||
default=15,
|
||||
help="Buffer trimming length threshold in seconds. If buffer length is longer, trimming sentence/segment is triggered.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-l",
|
||||
"--log-level",
|
||||
dest="log_level",
|
||||
choices=["DEBUG", "INFO", "WARNING", "ERROR", "CRITICAL"],
|
||||
help="Set the log level",
|
||||
default="DEBUG",
|
||||
)
|
||||
|
||||
|
||||
def asr_factory(args, logfile=sys.stderr):
|
||||
"""
|
||||
|
|
@ -789,12 +941,17 @@ def asr_factory(args, logfile=sys.stderr):
|
|||
size = args.model
|
||||
t = time.time()
|
||||
logger.info(f"Loading Whisper {size} model for {args.lan}...")
|
||||
asr = asr_cls(modelsize=size, lan=args.lan, cache_dir=args.model_cache_dir, model_dir=args.model_dir)
|
||||
asr = asr_cls(
|
||||
modelsize=size,
|
||||
lan=args.lan,
|
||||
cache_dir=args.model_cache_dir,
|
||||
model_dir=args.model_dir,
|
||||
)
|
||||
e = time.time()
|
||||
logger.info(f"done. It took {round(e-t,2)} seconds.")
|
||||
|
||||
# Apply common configurations
|
||||
if getattr(args, 'vad', False): # Checks if VAD argument is present and True
|
||||
if getattr(args, "vad", False): # Checks if VAD argument is present and True
|
||||
logger.info("Setting VAD filter")
|
||||
asr.use_vad()
|
||||
|
||||
|
|
@ -814,30 +971,59 @@ def asr_factory(args, logfile=sys.stderr):
|
|||
# Create the OnlineASRProcessor
|
||||
if args.vac:
|
||||
|
||||
online = VACOnlineASRProcessor(args.min_chunk_size, asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
|
||||
online = VACOnlineASRProcessor(
|
||||
args.min_chunk_size,
|
||||
asr,
|
||||
tokenizer,
|
||||
logfile=logfile,
|
||||
buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec),
|
||||
)
|
||||
else:
|
||||
online = OnlineASRProcessor(asr,tokenizer,logfile=logfile,buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec))
|
||||
online = OnlineASRProcessor(
|
||||
asr,
|
||||
tokenizer,
|
||||
logfile=logfile,
|
||||
buffer_trimming=(args.buffer_trimming, args.buffer_trimming_sec),
|
||||
)
|
||||
|
||||
return asr, online
|
||||
|
||||
|
||||
def set_logging(args, logger, other="_server"):
|
||||
logging.basicConfig(#format='%(name)s
|
||||
format='%(levelname)s\t%(message)s')
|
||||
logging.basicConfig(format="%(levelname)s\t%(message)s") # format='%(name)s
|
||||
logger.setLevel(args.log_level)
|
||||
logging.getLogger("whisper_online" + other).setLevel(args.log_level)
|
||||
# logging.getLogger("whisper_online_server").setLevel(args.log_level)
|
||||
|
||||
|
||||
# logging.getLogger("whisper_online_server").setLevel(args.log_level)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument('audio_path', type=str, help="Filename of 16kHz mono channel wav, on which live streaming is simulated.")
|
||||
parser.add_argument(
|
||||
"audio_path",
|
||||
type=str,
|
||||
help="Filename of 16kHz mono channel wav, on which live streaming is simulated.",
|
||||
)
|
||||
add_shared_args(parser)
|
||||
parser.add_argument('--start_at', type=float, default=0.0, help='Start processing audio at this time.')
|
||||
parser.add_argument('--offline', action="store_true", default=False, help='Offline mode.')
|
||||
parser.add_argument('--comp_unaware', action="store_true", default=False, help='Computationally unaware simulation.')
|
||||
parser.add_argument(
|
||||
"--start_at",
|
||||
type=float,
|
||||
default=0.0,
|
||||
help="Start processing audio at this time.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--offline", action="store_true", default=False, help="Offline mode."
|
||||
)
|
||||
parser.add_argument(
|
||||
"--comp_unaware",
|
||||
action="store_true",
|
||||
default=False,
|
||||
help="Computationally unaware simulation.",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
|
|
@ -845,7 +1031,9 @@ if __name__ == "__main__":
|
|||
logfile = sys.stderr
|
||||
|
||||
if args.offline and args.comp_unaware:
|
||||
logger.error("No or one option from --offline and --comp_unaware are available, not both. Exiting.")
|
||||
logger.error(
|
||||
"No or one option from --offline and --comp_unaware are available, not both. Exiting."
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# if args.log_level:
|
||||
|
|
@ -885,8 +1073,15 @@ if __name__ == "__main__":
|
|||
if now is None:
|
||||
now = time.time() - start
|
||||
if o[0] is not None:
|
||||
print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),file=logfile,flush=True)
|
||||
print("%1.4f %1.0f %1.0f %s" % (now*1000, o[0]*1000,o[1]*1000,o[2]),flush=True)
|
||||
print(
|
||||
"%1.4f %1.0f %1.0f %s" % (now * 1000, o[0] * 1000, o[1] * 1000, o[2]),
|
||||
file=logfile,
|
||||
flush=True,
|
||||
)
|
||||
print(
|
||||
"%1.4f %1.0f %1.0f %s" % (now * 1000, o[0] * 1000, o[1] * 1000, o[2]),
|
||||
flush=True,
|
||||
)
|
||||
else:
|
||||
# No text, so no output
|
||||
pass
|
||||
|
|
@ -946,7 +1141,9 @@ if __name__ == "__main__":
|
|||
else:
|
||||
output_transcript(o)
|
||||
now = time.time() - start
|
||||
logger.debug(f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}")
|
||||
logger.debug(
|
||||
f"## last processed {end:.2f} s, now is {now:.2f}, the latency is {now-end:.2f}"
|
||||
)
|
||||
|
||||
if end >= duration:
|
||||
break
|
||||
|
|
|
|||
Loading…
Reference in a new issue