admin-valentin/WhisperLiveKit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

stt/diar/nllw alignment: internal rework 5

Browse source
This commit is contained in:
Quentin Fuxa 2025-11-20 23:52:00 +01:00
parent f4f9831d39
commit b5067249c0
3 changed files with 22 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
whisperlivekit/audio_processor.py
View file

@ -361,21 +361,25 @@ class AudioProcessor:
# in the future we want to have different languages for each speaker etc, so it will be more complex. # in the future we want to have different languages for each speaker etc, so it will be more complex.
while True: while True:
try: try:
tokens_to_process = await get_all_from_queue(self.translation_queue) item = await get_all_from_queue(self.translation_queue)
if tokens_to_process is SENTINEL: if item is SENTINEL:
logger.debug("Translation processor received sentinel. Finishing.") logger.debug("Translation processor received sentinel. Finishing.")
self.translation_queue.task_done()
break break
elif type(tokens_to_process) is Silence: elif type(item) is Silence:
if tokens_to_process.has_ended: if item.is_starting:
self.translation.insert_silence(tokens_to_process.duration) new_translation, new_translation_buffer = self.translation.validate_buffer_and_reset()
continue if item.has_ended:
if tokens_to_process: self.translation.insert_silence(item.duration)
self.translation.insert_tokens(tokens_to_process) continue
translation_validated_segments, buffer_translation = await asyncio.to_thread(self.translation.process) elif isinstance(item, ChangeSpeaker):
async with self.lock: new_translation, new_translation_buffer = self.translation.validate_buffer_and_reset()
self.state.new_translation = translation_validated_segments pass
self.state.new_translation_buffer = buffer_translation else:
self.translation.insert_tokens(item)
new_translation, new_translation_buffer = await asyncio.to_thread(self.translation.process)
async with self.lock:
self.state.new_translation.append(new_translation)
self.state.new_translation_buffer = new_translation_buffer
except Exception as e: except Exception as e:
logger.warning(f"Exception in translation_processor: {e}") logger.warning(f"Exception in translation_processor: {e}")
logger.warning(f"Traceback: {traceback.format_exc()}") logger.warning(f"Traceback: {traceback.format_exc()}")

2
whisperlivekit/timed_objects.py
View file

@ -233,4 +233,4 @@ class State():
new_translation: List[Any] = field(default_factory=list) new_translation: List[Any] = field(default_factory=list)
new_diarization: List[Any] = field(default_factory=list) new_diarization: List[Any] = field(default_factory=list)
new_tokens_buffer: List[Any] = field(default_factory=list) # only when local agreement new_tokens_buffer: List[Any] = field(default_factory=list) # only when local agreement
new_translation_buffer: str = '' new_translation_buffer= TimedText()

10
whisperlivekit/tokens_alignment.py
View file

@ -33,15 +33,13 @@ class TokensAlignment:
self.all_tokens.extend(self.new_tokens) self.all_tokens.extend(self.new_tokens)
self.all_diarization_segments.extend(self.new_diarization) self.all_diarization_segments.extend(self.new_diarization)
# self.all_translation_segments.extend(self.new_translation) #future self.all_translation_segments.extend(self.new_translation)
self.all_translation_segments = self.new_translation if self.new_translation != [] else self.all_translation_segments self.new_translation_buffer = self.state.new_translation_buffer
self.new_translation_buffer = self.state.new_translation_buffer if self.new_translation else self.new_translation_buffer
self.new_translation_buffer = self.new_translation_buffer if type(self.new_translation_buffer) == str else self.new_translation_buffer.text
def add_translation(self, line: Line) -> None: def add_translation(self, line: Line) -> None:
for ts in self.all_translation_segments: for ts in self.all_translation_segments:
if ts.is_within(line): if ts.is_within(line):
line.translation += ts.text + self.sep line.translation += ts.text + (self.sep if ts.text else '')
elif line.translation: elif line.translation:
break break
@ -175,4 +173,4 @@ class TokensAlignment:
)) ))
if translation: if translation:
[self.add_translation(line) for line in lines if not type(line) == Silence] [self.add_translation(line) for line in lines if not type(line) == Silence]
return lines, diarization_buffer, self.new_translation_buffer return lines, diarization_buffer, self.new_translation_buffer.text