admin-valentin/WhisperLiveKit
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

audio processor lines use now Lines objects instead of dict

Browse source
This commit is contained in:
Quentin Fuxa 2025-09-09 21:45:00 +02:00
parent add7ea07ee
commit cb2d4ea88a
5 changed files with 73 additions and 65 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

6
chrome-extension/live_transcription.js
View file

@ -310,7 +310,7 @@ function renderLinesWithBuffer(
const showTransLag = !isFinalizing && remaining_time_transcription > 0; const showTransLag = !isFinalizing && remaining_time_transcription > 0;
const showDiaLag = !isFinalizing && !!buffer_diarization && remaining_time_diarization > 0; const showDiaLag = !isFinalizing && !!buffer_diarization && remaining_time_diarization > 0;
const signature = JSON.stringify({ const signature = JSON.stringify({
lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, beg: it.beg, end: it.end })), lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, start: it.start, end: it.end })),
buffer_transcription: buffer_transcription || "", buffer_transcription: buffer_transcription || "",
buffer_diarization: buffer_diarization || "", buffer_diarization: buffer_diarization || "",
status: current_status, status: current_status,
@ -333,8 +333,8 @@ function renderLinesWithBuffer(
const linesHtml = (lines || []) const linesHtml = (lines || [])
.map((item, idx) => { .map((item, idx) => {
let timeInfo = ""; let timeInfo = "";
if (item.beg !== undefined && item.end !== undefined) { if (item.start !== undefined && item.end !== undefined) {
timeInfo = ` ${item.beg} - ${item.end}`; timeInfo = ` ${item.start} - ${item.end}`;
} }
let speakerLabel = ""; let speakerLabel = "";

38
whisperlivekit/audio_processor.py
View file

@ -4,11 +4,11 @@ from time import time, sleep
import math import math
import logging import logging
import traceback import traceback
from whisperlivekit.timed_objects import ASRToken, Silence from whisperlivekit.timed_objects import ASRToken, Silence, Line
from whisperlivekit.core import TranscriptionEngine, online_factory, online_diarization_factory, online_translation_factory from whisperlivekit.core import TranscriptionEngine, online_factory, online_diarization_factory, online_translation_factory
from whisperlivekit.ffmpeg_manager import FFmpegManager, FFmpegState from whisperlivekit.ffmpeg_manager import FFmpegManager, FFmpegState
from whisperlivekit.silero_vad_iterator import FixedVADIterator from whisperlivekit.silero_vad_iterator import FixedVADIterator
from whisperlivekit.results_formater import format_output, format_time from whisperlivekit.results_formater import format_output
# Set up logging once # Set up logging once
logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s") logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
@ -59,7 +59,7 @@ class AudioProcessor:
self.silence = False self.silence = False
self.silence_duration = 0.0 self.silence_duration = 0.0
self.tokens = [] self.tokens = []
self.translated_tokens = [] self.translated_segments = []
self.buffer_transcription = "" self.buffer_transcription = ""
self.buffer_diarization = "" self.buffer_diarization = ""
self.end_buffer = 0 self.end_buffer = 0
@ -153,7 +153,7 @@ class AudioProcessor:
return { return {
"tokens": self.tokens.copy(), "tokens": self.tokens.copy(),
"translated_tokens": self.translated_tokens.copy(), "translated_segments": self.translated_segments.copy(),
"buffer_transcription": self.buffer_transcription, "buffer_transcription": self.buffer_transcription,
"buffer_diarization": self.buffer_diarization, "buffer_diarization": self.buffer_diarization,
"end_buffer": self.end_buffer, "end_buffer": self.end_buffer,
@ -167,7 +167,7 @@ class AudioProcessor:
"""Reset all state variables to initial values.""" """Reset all state variables to initial values."""
async with self.lock: async with self.lock:
self.tokens = [] self.tokens = []
self.translated_tokens = [] self.translated_segments = []
self.buffer_transcription = self.buffer_diarization = "" self.buffer_transcription = self.buffer_diarization = ""
self.end_buffer = self.end_attributed_speaker = 0 self.end_buffer = self.end_attributed_speaker = 0
self.beg_loop = time() self.beg_loop = time()
@ -431,8 +431,7 @@ class AudioProcessor:
tokens_to_process.append(additional_token) tokens_to_process.append(additional_token)
if tokens_to_process: if tokens_to_process:
online_translation.insert_tokens(tokens_to_process) online_translation.insert_tokens(tokens_to_process)
translations = online_translation.process() self.translated_segments = online_translation.process()
print(translations)
self.translation_queue.task_done() self.translation_queue.task_done()
for _ in additional_tokens: for _ in additional_tokens:
@ -505,23 +504,19 @@ class AudioProcessor:
buffer_diarization = combined buffer_diarization = combined
response_status = "active_transcription" response_status = "active_transcription"
final_lines_for_response = lines.copy()
if not tokens and not buffer_transcription and not buffer_diarization: if not tokens and not buffer_transcription and not buffer_diarization:
response_status = "no_audio_detected" response_status = "no_audio_detected"
final_lines_for_response = [] lines = []
elif response_status == "active_transcription" and not final_lines_for_response: elif response_status == "active_transcription" and not lines:
final_lines_for_response = [{ lines = [Line(
"speaker": 1, speaker=1,
"text": "", start=state.get("end_buffer", 0),
"beg": format_time(state.get("end_buffer", 0)), end=state.get("end_buffer", 0)
"end": format_time(state.get("end_buffer", 0)), )]
"diff": 0
}]
response = { response = {
"status": response_status, "status": response_status,
"lines": final_lines_for_response, "lines": [line.to_dict() for line in lines],
"buffer_transcription": buffer_transcription, "buffer_transcription": buffer_transcription,
"buffer_diarization": buffer_diarization, "buffer_diarization": buffer_diarization,
"remaining_time_transcription": state["remaining_time_transcription"], "remaining_time_transcription": state["remaining_time_transcription"],
@ -529,7 +524,7 @@ class AudioProcessor:
} }
current_response_signature = f"{response_status} | " + \ current_response_signature = f"{response_status} | " + \
' '.join([f"{line['speaker']} {line['text']}" for line in final_lines_for_response]) + \ ' '.join([f"{line.speaker} {line.text}" for line in lines]) + \
f" | {buffer_transcription} | {buffer_diarization}" f" | {buffer_transcription} | {buffer_diarization}"
trans = state["remaining_time_transcription"] trans = state["remaining_time_transcription"]
@ -540,7 +535,7 @@ class AudioProcessor:
or round(trans, 1) != round(last_sent_trans, 1) or round(trans, 1) != round(last_sent_trans, 1)
or round(diar, 1) != round(last_sent_diar, 1) or round(diar, 1) != round(last_sent_diar, 1)
) )
if should_push and (final_lines_for_response or buffer_transcription or buffer_diarization or response_status == "no_audio_detected" or trans > 0 or diar > 0): if should_push and (lines or buffer_transcription or buffer_diarization or response_status == "no_audio_detected" or trans > 0 or diar > 0):
yield response yield response
self.last_response_content = current_response_signature self.last_response_content = current_response_signature
last_sent_trans = trans last_sent_trans = trans
@ -556,7 +551,6 @@ class AudioProcessor:
if all_processors_done: if all_processors_done:
logger.info("Results formatter: All upstream processors are done and in stopping state. Terminating.") logger.info("Results formatter: All upstream processors are done and in stopping state. Terminating.")
final_state = await self.get_current_state()
return return
await asyncio.sleep(0.1) # Avoid overwhelming the client await asyncio.sleep(0.1) # Avoid overwhelming the client

66
whisperlivekit/results_formater.py
View file

@ -1,7 +1,7 @@
import logging import logging
from datetime import timedelta
from whisperlivekit.remove_silences import handle_silences from whisperlivekit.remove_silences import handle_silences
from timed_objects import Line, format_time
logger = logging.getLogger(__name__) logger = logging.getLogger(__name__)
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
@ -9,11 +9,6 @@ logger.setLevel(logging.DEBUG)
PUNCTUATION_MARKS = {'.', '!', '?', '。', '！', '？'} PUNCTUATION_MARKS = {'.', '!', '?', '。', '！', '？'}
CHECK_AROUND = 4 CHECK_AROUND = 4
def format_time(seconds: float) -> str:
"""Format seconds as HH:MM:SS."""
return str(timedelta(seconds=int(seconds)))
def is_punctuation(token): def is_punctuation(token):
if token.text.strip() in PUNCTUATION_MARKS: if token.text.strip() in PUNCTUATION_MARKS:
return True return True
@ -34,32 +29,26 @@ def next_speaker_change(i, tokens, speaker):
return ind, token.speaker return ind, token.speaker
return None, speaker return None, speaker
def new_line( def new_line(
token, token,
speaker, speaker,
last_end_diarized,
debug_info = "" debug_info = ""
): ):
return { return Line(
"speaker": int(speaker), speaker = speaker,
"text": token.text + debug_info, text = token.text + debug_info,
"beg": format_time(token.start), start = token.start,
"end": format_time(token.end), end = token.end,
"diff": round(token.end - last_end_diarized, 2) )
}
def append_token_to_last_line(lines, sep, token, debug_info):
def append_token_to_last_line(lines, sep, token, debug_info, last_end_diarized):
if token.text: if token.text:
lines[-1]["text"] += sep + token.text + debug_info lines[-1].text += sep + token.text + debug_info
lines[-1]["end"] = format_time(token.end) lines[-1].end = token.end
lines[-1]["diff"] = round(token.end - last_end_diarized, 2)
def format_output(state, silence, current_time, diarization, debug): def format_output(state, silence, current_time, diarization, debug):
tokens = state["tokens"] tokens = state["tokens"]
translated_tokens = state["translated_tokens"] # Here we will attribute the speakers only based on the timestamps of the segments translated_segments = state["translated_segments"] # Here we will attribute the speakers only based on the timestamps of the segments
buffer_transcription = state["buffer_transcription"] buffer_transcription = state["buffer_transcription"]
buffer_diarization = state["buffer_diarization"] buffer_diarization = state["buffer_diarization"]
end_attributed_speaker = state["end_attributed_speaker"] end_attributed_speaker = state["end_attributed_speaker"]
@ -67,13 +56,11 @@ def format_output(state, silence, current_time, diarization, debug):
previous_speaker = -1 previous_speaker = -1
lines = [] lines = []
last_end_diarized = 0
undiarized_text = [] undiarized_text = []
tokens, buffer_transcription, buffer_diarization = handle_silences(tokens, buffer_transcription, buffer_diarization, current_time, silence) tokens, buffer_transcription, buffer_diarization = handle_silences(tokens, buffer_transcription, buffer_diarization, current_time, silence)
last_punctuation = None last_punctuation = None
for i, token in enumerate(tokens): for i, token in enumerate(tokens):
speaker = token.speaker speaker = token.speaker
if not diarization and speaker == -1: #Speaker -1 means no attributed by diarization. In the frontend, it should appear under 'Speaker 1' if not diarization and speaker == -1: #Speaker -1 means no attributed by diarization. In the frontend, it should appear under 'Speaker 1'
speaker = 1 speaker = 1
if diarization and not tokens[-1].speaker == -2: if diarization and not tokens[-1].speaker == -2:
@ -82,18 +69,15 @@ def format_output(state, silence, current_time, diarization, debug):
continue continue
elif (speaker in [-1, 0]) and token.end < end_attributed_speaker: elif (speaker in [-1, 0]) and token.end < end_attributed_speaker:
speaker = previous_speaker speaker = previous_speaker
if speaker not in [-1, 0]:
last_end_diarized = max(token.end, last_end_diarized)
debug_info = "" debug_info = ""
if debug: if debug:
debug_info = f"[{format_time(token.start)} : {format_time(token.end)}]" debug_info = f"[{format_time(token.start)} : {format_time(token.end)}]"
if not lines: if not lines:
lines.append(new_line(token, speaker, last_end_diarized, debug_info = "")) lines.append(new_line(token, speaker, debug_info = ""))
continue continue
else: else:
previous_speaker = lines[-1]['speaker'] previous_speaker = lines[-1].speaker
if is_punctuation(token): if is_punctuation(token):
last_punctuation = i last_punctuation = i
@ -102,7 +86,7 @@ def format_output(state, silence, current_time, diarization, debug):
if last_punctuation == i-1: if last_punctuation == i-1:
if speaker != previous_speaker: if speaker != previous_speaker:
# perfect, diarization perfectly aligned # perfect, diarization perfectly aligned
lines.append(new_line(token, speaker, last_end_diarized, debug_info = "")) lines.append(new_line(token, speaker, debug_info = ""))
last_punctuation, next_punctuation = None, None last_punctuation, next_punctuation = None, None
continue continue
@ -112,28 +96,38 @@ def format_output(state, silence, current_time, diarization, debug):
# That was the idea. Okay haha |SPLIT SPEAKER| that's a good one # That was the idea. Okay haha |SPLIT SPEAKER| that's a good one
# should become: # should become:
# That was the idea. |SPLIT SPEAKER| Okay haha that's a good one # That was the idea. |SPLIT SPEAKER| Okay haha that's a good one
lines.append(new_line(token, new_speaker, last_end_diarized, debug_info = "")) lines.append(new_line(token, new_speaker, debug_info = ""))
else: else:
# No speaker change to come # No speaker change to come
append_token_to_last_line(lines, sep, token, debug_info, last_end_diarized) append_token_to_last_line(lines, sep, token, debug_info)
continue continue
if speaker != previous_speaker: if speaker != previous_speaker:
if speaker == -2 or previous_speaker == -2: #silences can happen anytime if speaker == -2 or previous_speaker == -2: #silences can happen anytime
lines.append(new_line(token, speaker, last_end_diarized, debug_info = "")) lines.append(new_line(token, speaker, debug_info = ""))
continue continue
elif next_punctuation_change(i, tokens): elif next_punctuation_change(i, tokens):
# Corrects advance: # Corrects advance:
# Are you |SPLIT SPEAKER| okay? yeah, sure. Absolutely # Are you |SPLIT SPEAKER| okay? yeah, sure. Absolutely
# should become: # should become:
# Are you okay? |SPLIT SPEAKER| yeah, sure. Absolutely # Are you okay? |SPLIT SPEAKER| yeah, sure. Absolutely
append_token_to_last_line(lines, sep, token, debug_info, last_end_diarized) append_token_to_last_line(lines, sep, token, debug_info)
continue continue
else: #we create a new speaker, but that's no ideal. We are not sure about the split. We prefer to append to previous line else: #we create a new speaker, but that's no ideal. We are not sure about the split. We prefer to append to previous line
# lines.append(new_line(token, speaker, last_end_diarized, debug_info = "")) # lines.append(new_line(token, speaker, debug_info = ""))
pass pass
append_token_to_last_line(lines, sep, token, debug_info, last_end_diarized) append_token_to_last_line(lines, sep, token, debug_info)
if lines and translated_segments:
cts_idx = 0 # current_translated_segment_idx
for line in lines:
while cts_idx < len(translated_segments):
ts = translated_segments[cts_idx]
if ts.start and ts.start >= line.start and ts.end <= line.end:
line.translation += ts.text + ' '
cts_idx += 1
else:
break
return lines, undiarized_text, buffer_transcription, '' return lines, undiarized_text, buffer_transcription, ''

22
whisperlivekit/timed_objects.py
View file

@ -1,5 +1,11 @@
from dataclasses import dataclass from dataclasses import dataclass
from typing import Optional from typing import Optional
from datetime import timedelta
def format_time(seconds: float) -> str:
"""Format seconds as HH:MM:SS."""
return str(timedelta(seconds=int(seconds)))
@dataclass @dataclass
class TimedText: class TimedText:
@ -37,4 +43,18 @@ class Translation(TimedText):
@dataclass @dataclass
class Silence(): class Silence():
duration: float duration: float
@dataclass
class Line(TimedText):
translation: str = ''
def to_dict(self):
return {
'speaker': int(self.speaker),
'text': self.text,
'translation': self.translation,
'start': format_time(self.start),
'end': format_time(self.end),
}

6
whisperlivekit/web/live_transcription.js
View file

@ -293,7 +293,7 @@ function renderLinesWithBuffer(
const showTransLag = !isFinalizing && remaining_time_transcription > 0; const showTransLag = !isFinalizing && remaining_time_transcription > 0;
const showDiaLag = !isFinalizing && !!buffer_diarization && remaining_time_diarization > 0; const showDiaLag = !isFinalizing && !!buffer_diarization && remaining_time_diarization > 0;
const signature = JSON.stringify({ const signature = JSON.stringify({
lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, beg: it.beg, end: it.end })), lines: (lines || []).map((it) => ({ speaker: it.speaker, text: it.text, start: it.start, end: it.end })),
buffer_transcription: buffer_transcription || "", buffer_transcription: buffer_transcription || "",
buffer_diarization: buffer_diarization || "", buffer_diarization: buffer_diarization || "",
status: current_status, status: current_status,
@ -316,8 +316,8 @@ function renderLinesWithBuffer(
const linesHtml = (lines || []) const linesHtml = (lines || [])
.map((item, idx) => { .map((item, idx) => {
let timeInfo = ""; let timeInfo = "";
if (item.beg !== undefined && item.end !== undefined) { if (item.start !== undefined && item.end !== undefined) {
timeInfo = ` ${item.beg} - ${item.end}`; timeInfo = ` ${item.start} - ${item.end}`;
} }
let speakerLabel = ""; let speakerLabel = "";