admin-valentin/WhisperLiveKit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Make --vad work with --backend openai-api

Browse source
This commit is contained in:
Tijs Zwinkels 2024-02-10 15:29:18 +01:00
parent 3696fef2b1
commit f0a24cd5e1
1 changed files with 22 additions and 16 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

38
whisper_online.py
View file

@ -162,7 +162,7 @@ class OpenaiApiASR(ASRBase):
self.load_model() self.load_model()
self.use_vad = False self.use_vad_opt = False
# reset the task in set_translate_task # reset the task in set_translate_task
self.task = "transcribe" self.task = "transcribe"
@ -175,21 +175,27 @@ class OpenaiApiASR(ASRBase):
def ts_words(self, segments): def ts_words(self, segments):
o = [] no_speech_segments = []
# If VAD on, skip segments containing no speech. if self.use_vad_opt:
# TODO: threshold can be set from outside for segment in segments.segments:
# TODO: Make VAD work again with word-level timestamps # TODO: threshold can be set from outside
#if self.use_vad and segment["no_speech_prob"] > 0.8: if segment["no_speech_prob"] > 0.8:
# continue no_speech_segments.append((segment.get("start"), segment.get("end")))
for word in segments: o = []
o.append((word.get("start"), word.get("end"), word.get("word"))) for word in segments.words:
start = word.get("start")
end = word.get("end")
if any(s[0] <= start <= s[1] for s in no_speech_segments):
# print("Skipping word", word.get("word"), "because it's in a no-speech segment")
continue
o.append((start, end, word.get("word")))
return o return o
def segments_end_ts(self, res): def segments_end_ts(self, res):
return [s["end"] for s in res] return [s["end"] for s in res.words]
def transcribe(self, audio_data, prompt=None, *args, **kwargs): def transcribe(self, audio_data, prompt=None, *args, **kwargs):
# Write the audio data to a buffer # Write the audio data to a buffer
@ -205,7 +211,7 @@ class OpenaiApiASR(ASRBase):
"file": buffer, "file": buffer,
"response_format": self.response_format, "response_format": self.response_format,
"temperature": self.temperature, "temperature": self.temperature,
"timestamp_granularities": ["word"] "timestamp_granularities": ["word", "segment"]
} }
if self.task != "translate" and self.language: if self.task != "translate" and self.language:
params["language"] = self.language params["language"] = self.language
@ -221,10 +227,10 @@ class OpenaiApiASR(ASRBase):
transcript = proc.create(**params) transcript = proc.create(**params)
print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile) print(f"OpenAI API processed accumulated {self.transcribed_seconds} seconds",file=self.logfile)
return transcript.words return transcript
def use_vad(self): def use_vad(self):
self.use_vad = True self.use_vad_opt = True
def set_translate_task(self): def set_translate_task(self):
self.task = "translate" self.task = "translate"
@ -592,9 +598,9 @@ if __name__ == "__main__":
e = time.time() e = time.time()
print(f"done. It took {round(e-t,2)} seconds.",file=logfile) print(f"done. It took {round(e-t,2)} seconds.",file=logfile)
if args.vad: if args.vad:
print("setting VAD filter",file=logfile) print("setting VAD filter",file=logfile)
asr.use_vad() asr.use_vad()
if args.task == "translate": if args.task == "translate":
asr.set_translate_task() asr.set_translate_task()