yjyoon / whisper_streaming_Deprecated star
Rodrigo 2023-12-02
vad
@5b1edda944f525cf51a567a35d666c97872f2c4c
 
5b1edda
mic_test_whisper_simple.py (added)
+++ mic_test_whisper_simple.py
@@ -0,0 +1,95 @@
+from microphone_stream import MicrophoneStream
+from voice_activity_controller import VoiceActivityController
+from whisper_online import *
+import numpy as np
+import librosa  
+import io
+import soundfile
+import sys
+
+
+
+
+class SimpleASRProcessor:
+
+    def __init__(self, asr, sampling_rate = 16000):
+        """run this when starting or restarting processing"""
+        self.audio_buffer = np.array([],dtype=np.float32)
+        self.prompt_buffer = ""
+        self.asr = asr
+        self.sampling_rate = sampling_rate
+        self.init_prompt = ''
+
+    def ts_words(self, segments):
+        result = ""
+        for segment in segments:
+            if segment.no_speech_prob > 0.9:
+                continue
+            for word in segment.words:
+                w = word.word
+                t = (word.start, word.end, w)
+                result +=w
+        return result 
+
+    def stream_process(self, vad_result):
+        iter_in_phrase = 0
+        for chunk, is_final in vad_result:
+            iter_in_phrase += 1
+
+            if chunk is not None:
+                sf = soundfile.SoundFile(io.BytesIO(chunk), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW")
+                audio, _ = librosa.load(sf,sr=SAMPLING_RATE)
+                # self.audio_buffer.append(chunk)
+                out = []
+                out.append(audio)
+                a = np.concatenate(out)
+                self.audio_buffer = np.append(self.audio_buffer, a)
+
+            if is_final and len(self.audio_buffer) > 0:
+                res = self.asr.transcribe(self.audio_buffer, init_prompt=self.init_prompt)
+                # use custom ts_words
+                tsw = self.ts_words(res)
+                self.init_prompt = self.init_prompt + tsw
+                self.init_prompt  = self.init_prompt [-100:]
+                self.audio_buffer.resize(0)
+                iter_in_phrase =0
+                yield True, tsw
+            # show progress evry 10 chunks
+            elif iter_in_phrase % 20 == 0 and len(self.audio_buffer) > 0:
+                res = self.asr.transcribe(self.audio_buffer, init_prompt=self.init_prompt)
+                # use custom ts_words
+                tsw = self.ts_words(res)
+                yield False, tsw
+            
+        
+
+
+
+
+
+SAMPLING_RATE = 16000
+
+model = "large-v2"
+src_lan = "en"  # source language
+tgt_lan = "en"  # target language  -- same as source for ASR, "en" if translate task is used
+use_vad_result = True
+min_sample_length = 1 * SAMPLING_RATE
+
+
+
+vad = VoiceActivityController(use_vad_result = use_vad_result)
+asr = FasterWhisperASR(src_lan, "large-v2")  # loads and wraps Whisper model
+
+tokenizer = create_tokenizer(tgt_lan)
+online = SimpleASRProcessor(asr)
+
+
+stream = MicrophoneStream()
+stream = vad.detect_user_speech(stream, audio_in_int16 = False) 
+stream = online.stream_process(stream)
+
+for isFinal, text in stream:
+    if isFinal:
+        print( text,  end="\r\n")
+    else:
+        print( text,  end="\r")
 
5b1edda
mic_test_whisper_streaming.py (added)
+++ mic_test_whisper_streaming.py
@@ -0,0 +1,71 @@
+from microphone_stream import MicrophoneStream
+from voice_activity_controller import VoiceActivityController
+from whisper_online import *
+import numpy as np
+import librosa  
+import io
+import soundfile
+import sys
+
+
+SAMPLING_RATE = 16000
+model = "large-v2"
+src_lan = "en"  # source language
+tgt_lan = "en"  # target language  -- same as source for ASR, "en" if translate task is used
+use_vad_result = True
+min_sample_length = 1 * SAMPLING_RATE
+
+
+
+asr = FasterWhisperASR(src_lan, model)  # loads and wraps Whisper model
+tokenizer = create_tokenizer(tgt_lan)  # sentence segmenter for the target language
+online = OnlineASRProcessor(asr, tokenizer)  # create processing object
+
+microphone_stream = MicrophoneStream() 
+vad = VoiceActivityController(use_vad_result = use_vad_result)
+
+complete_text = ''
+final_processing_pending = False
+out = []
+out_len = 0
+for iter in vad.detect_user_speech(microphone_stream):   # processing loop:
+    raw_bytes=  iter[0]
+    is_final =  iter[1]
+
+    if  raw_bytes:
+        sf = soundfile.SoundFile(io.BytesIO(raw_bytes), channels=1,endian="LITTLE",samplerate=SAMPLING_RATE, subtype="PCM_16",format="RAW")
+        audio, _ = librosa.load(sf,sr=SAMPLING_RATE)
+        out.append(audio)
+        out_len += len(audio)
+
+    
+    if (is_final or out_len >= min_sample_length) and out_len>0:
+        a = np.concatenate(out)
+        online.insert_audio_chunk(a)    
+        
+    if out_len > min_sample_length:
+        o = online.process_iter()
+        print('-----'*10)
+        complete_text = complete_text + o[2]
+        print('PARTIAL - '+ complete_text) # do something with current partial output
+        print('-----'*10)     
+        out = []
+        out_len = 0   
+
+    if is_final:
+        o = online.finish()
+        online.init()   
+        # final_processing_pending = False         
+        print('-----'*10)
+        complete_text = complete_text + o[2]
+        print('FINAL - '+ complete_text) # do something with current partial output
+        print('-----'*10)   
+        out = []
+        out_len = 0    
+        
+
+
+
+
+
+
 
5b1edda
microphone_stream.py (added)
+++ microphone_stream.py
@@ -0,0 +1,82 @@
+
+
+### mic stream
+
+import queue
+import re
+import sys
+import pyaudio
+
+
+class MicrophoneStream:
+    def __init__(
+        self,
+        sample_rate: int = 16000,
+    ):
+        """
+        Creates a stream of audio from the microphone.
+
+        Args:
+            chunk_size: The size of each chunk of audio to read from the microphone.
+            channels: The number of channels to record audio from.
+            sample_rate: The sample rate to record audio at.
+        """
+        try:
+            import pyaudio
+        except ImportError:
+            raise Exception('py audio not installed')
+
+        self._pyaudio = pyaudio.PyAudio()
+        self.sample_rate = sample_rate
+
+        self._chunk_size = int(self.sample_rate * 0.1)
+        self._stream = self._pyaudio.open(
+            format=pyaudio.paInt16,
+            channels=1,
+            rate=sample_rate,
+            input=True,
+            frames_per_buffer=self._chunk_size,
+        )
+
+        self._open = True
+
+    def __iter__(self):
+        """
+        Returns the iterator object.
+        """
+
+        return self
+
+    def __next__(self):
+        """
+        Reads a chunk of audio from the microphone.
+        """
+        if not self._open:
+            raise StopIteration
+
+        try:
+            return self._stream.read(self._chunk_size)
+        except KeyboardInterrupt:
+            raise StopIteration
+
+    def close(self):
+        """
+        Closes the stream.
+        """
+
+        self._open = False
+
+        if self._stream.is_active():
+            self._stream.stop_stream()
+
+        self._stream.close()
+        self._pyaudio.terminate()
+
+
+
+
+
+
+
+
+
 
5b1edda
voice_activity_controller.py (added)
+++ voice_activity_controller.py
@@ -0,0 +1,117 @@
+import torch
+import numpy as np
+# import sounddevice as sd
+import torch
+import numpy as np
+
+
+class VoiceActivityController:
+    def __init__(
+            self, 
+            sampling_rate = 16000,
+            second_ofSilence = 0.5,
+            second_ofSpeech = 0.25,
+            second_ofMinRecording = 10,
+            use_vad_result = True,
+            activity_detected_callback=None,
+        ):
+        self.activity_detected_callback=activity_detected_callback
+        self.model, self.utils = torch.hub.load(
+            repo_or_dir='snakers4/silero-vad',
+            model='silero_vad'
+        )
+        (self.get_speech_timestamps,
+        save_audio,
+        read_audio,
+        VADIterator,
+        collect_chunks) = self.utils
+
+        self.sampling_rate = sampling_rate  
+        self.silence_limit = second_ofSilence * self.sampling_rate 
+        self.speech_limit = second_ofSpeech *self.sampling_rate 
+        self.MIN_RECORDING_LENGTH =  second_ofMinRecording * self.sampling_rate 
+
+        self.use_vad_result = use_vad_result
+        self.vad_iterator = VADIterator(
+            model =self.model,
+            threshold = 0.3,
+            sampling_rate= 16000,
+            min_silence_duration_ms = 500, #100
+            speech_pad_ms = 400 #30
+        )
+        self.last_marked_chunk = None
+        
+    
+    def int2float(self, sound):
+        abs_max = np.abs(sound).max()
+        sound = sound.astype('float32')
+        if abs_max > 0:
+            sound *= 1/32768
+        sound = sound.squeeze()  # depends on the use case
+        return sound
+
+    def apply_vad(self, audio):
+        audio_float32 = self.int2float(audio)
+        chunk = self.vad_iterator(audio_float32, return_seconds=False)
+
+        if chunk is not None:        
+            if "start" in chunk:
+                start = chunk["start"]
+                self.last_marked_chunk = chunk
+                return audio[start:] if self.use_vad_result else audio, (len(audio) - start), 0
+            
+            if "end" in chunk:
+                #todo: pending get the padding from the next chunk
+                end = chunk["end"] if chunk["end"] < len(audio) else len(audio)
+                self.last_marked_chunk = chunk
+                return audio[:end] if self.use_vad_result else audio, end ,len(audio) - end
+
+        if self.last_marked_chunk is not None:
+            if "start" in self.last_marked_chunk:
+                return audio, len(audio)  ,0
+
+            if "end" in self.last_marked_chunk:
+                return  np.array([], dtype=np.float16) if self.use_vad_result else audio, 0 ,len(audio) 
+
+        return  np.array([], dtype=np.float16) if self.use_vad_result else audio, 0 , 0 
+
+
+
+    def detect_user_speech(self, audio_stream, audio_in_int16 = False):
+        silence_len= 0
+        speech_len = 0
+
+        for data in audio_stream:  # replace with your condition of choice
+            # if isinstance(data, EndOfTransmission):
+            #     raise EndOfTransmission("End of transmission detected")
+            
+            
+            audio_block = np.frombuffer(data, dtype=np.int16) if not audio_in_int16 else data
+            wav = audio_block
+            
+
+            is_final = False
+            voice_audio, speech_in_wav, last_silent_duration_in_wav = self.apply_vad(wav)
+            # print(f'----r> speech_in_wav: {speech_in_wav} last_silent_duration_in_wav: {last_silent_duration_in_wav}')
+
+            if speech_in_wav > 0 :
+                silence_len= 0                
+                speech_len += speech_in_wav
+                if self.activity_detected_callback is not None:
+                    self.activity_detected_callback()
+
+            silence_len = silence_len + last_silent_duration_in_wav
+            if silence_len>= self.silence_limit and speech_len >= self.speech_limit:
+                is_final = True
+                silence_len= 0
+                speech_len = 0
+            
+
+            yield voice_audio.tobytes(), is_final
+
+
+
+
+
+
+
Add a comment
List