import numpy as np import torch import logging from whisperlivekit.timed_objects import SpeakerSegment logger = logging.getLogger(__name__) try: from nemo.collections.asr.models import SortformerEncLabelModel except ImportError: raise SystemExit("""Please use `pip install "git+https://github.com/NVIDIA/NeMo.git@main#egg=nemo_toolkit[asr]"` to use the Sortformer diarization""") class SortformerDiarization: def __init__(self, model_name="nvidia/diar_streaming_sortformer_4spk-v2"): self.diar_model = SortformerEncLabelModel.from_pretrained(model_name) self.diar_model.eval() if torch.cuda.is_available(): self.diar_model.to(torch.device("cuda")) # Streaming parameters for speed self.diar_model.sortformer_modules.chunk_len = 12 self.diar_model.sortformer_modules.chunk_right_context = 1 self.diar_model.sortformer_modules.spkcache_len = 188 self.diar_model.sortformer_modules.fifo_len = 188 self.diar_model.sortformer_modules.spkcache_update_period = 144 self.diar_model.sortformer_modules.log = False self.diar_model.sortformer_modules._check_streaming_parameters() self.batch_size = 1 self.processed_signal_offset = torch.zeros((self.batch_size,), dtype=torch.long, device=self.diar_model.device) self.audio_buffer = np.array([], dtype=np.float32) self.sample_rate = 16000 self.speaker_segments = [] self.streaming_state = self.diar_model.sortformer_modules.init_streaming_state( batch_size=self.batch_size, async_streaming=True, device=self.diar_model.device ) self.total_preds = torch.zeros((self.batch_size, 0, self.diar_model.sortformer_modules.n_spk), device=self.diar_model.device) def _prepare_audio_signal(self, signal): audio_signal = torch.tensor(signal).unsqueeze(0).to(self.diar_model.device) audio_signal_length = torch.tensor([audio_signal.shape[1]]).to(self.diar_model.device) processed_signal, processed_signal_length = self.diar_model.preprocessor(input_signal=audio_signal, length=audio_signal_length) return processed_signal, processed_signal_length def _create_streaming_loader(self, processed_signal, processed_signal_length): streaming_loader = self.diar_model.sortformer_modules.streaming_feat_loader( feat_seq=processed_signal, feat_seq_length=processed_signal_length, feat_seq_offset=self.processed_signal_offset, ) return streaming_loader async def diarize(self, pcm_array: np.ndarray): """ Process an incoming audio chunk for diarization. """ self.audio_buffer = np.concatenate([self.audio_buffer, pcm_array]) # Process in fixed-size chunks (e.g., 1 second) chunk_size = self.sample_rate # 1 second of audio while len(self.audio_buffer) >= chunk_size: chunk_to_process = self.audio_buffer[:chunk_size] self.audio_buffer = self.audio_buffer[chunk_size:] processed_signal, processed_signal_length = self._prepare_audio_signal(chunk_to_process) current_offset_seconds = self.processed_signal_offset.item() * self.diar_model.preprocessor._cfg.window_stride streaming_loader = self._create_streaming_loader(processed_signal, processed_signal_length) frame_duration_s = self.diar_model.sortformer_modules.subsampling_factor * self.diar_model.preprocessor._cfg.window_stride chunk_duration_seconds = self.diar_model.sortformer_modules.chunk_len * frame_duration_s for i, chunk_feat_seq_t, feat_lengths, left_offset, right_offset in streaming_loader: with torch.inference_mode(): self.streaming_state, self.total_preds = self.diar_model.forward_streaming_step( processed_signal=chunk_feat_seq_t, processed_signal_length=feat_lengths, streaming_state=self.streaming_state, total_preds=self.total_preds, left_offset=left_offset, right_offset=right_offset, ) num_new_frames = feat_lengths[0].item() # Get predictions for the current chunk from the end of total_preds preds_np = self.total_preds[0, -num_new_frames:].cpu().numpy() active_speakers = np.argmax(preds_np, axis=1) for idx, spk in enumerate(active_speakers): start_time = current_offset_seconds + (i * chunk_duration_seconds) + (idx * frame_duration_s) end_time = start_time + frame_duration_s if self.speaker_segments and self.speaker_segments[-1].speaker == spk + 1: self.speaker_segments[-1].end = end_time else: self.speaker_segments.append(SpeakerSegment( speaker=int(spk + 1), start=start_time, end=end_time )) self.processed_signal_offset += processed_signal_length def assign_speakers_to_tokens(self, tokens: list, **kwargs) -> list: """ Assign speakers to tokens based on timing overlap with speaker segments. """ for token in tokens: for segment in self.speaker_segments: if not (segment.end <= token.start or segment.start >= token.end): token.speaker = segment.speaker return tokens def close(self): """ Cleanup resources. """ logger.info("Closing SortformerDiarization.") if __name__ == '__main__': import librosa an4_audio = 'new_audio_test.mp3' signal, sr = librosa.load(an4_audio, sr=16000) diarization_pipeline = SortformerDiarization() # Simulate streaming chunk_size = 16000 # 1 second for i in range(0, len(signal), chunk_size): chunk = signal[i:i+chunk_size] import asyncio asyncio.run(diarization_pipeline.diarize(chunk)) for segment in diarization_pipeline.speaker_segments: print(f"Speaker {segment.speaker}: {segment.start:.2f}s - {segment.end:.2f}s")