Commit @b9c9ba8472852eea8e690d51caee53e274f40c25 - yjyoon/whisper_server

Fedir Zadniprovskyi 01-10

chore: use async handlers in gradio

@b9c9ba8472852eea8e690d51caee53e274f40c25

9cf1e38

b9c9ba8

src/faster_whisper_server/gradio_app.py

--- src/faster_whisper_server/gradio_app.py

+++ src/faster_whisper_server/gradio_app.py


-from collections.abc import Generator
+from collections.abc import AsyncGenerator
 from pathlib import Path
 
 import gradio as gr
 import httpx
-from httpx_sse import connect_sse
-from openai import OpenAI
+from httpx_sse import aconnect_sse
+from openai import AsyncOpenAI
 
 from faster_whisper_server.config import Config, Task
 from faster_whisper_server.hf_utils import PiperModel

 
 def create_gradio_demo(config: Config) -> gr.Blocks:  # noqa: C901, PLR0915
     base_url = f"http://{config.host}:{config.port}"
-    http_client = httpx.Client(base_url=base_url, timeout=TIMEOUT)
-    openai_client = OpenAI(base_url=f"{base_url}/v1", api_key="cant-be-empty")
+    # TODO: test that auth works
+    http_client = httpx.AsyncClient(
+        base_url=base_url,
+        timeout=TIMEOUT,
+        headers={"Authorization": f"Bearer {config.api_key}"} if config.api_key else {},
+    )
+    openai_client = AsyncOpenAI(
+        base_url=f"{base_url}/v1", api_key=config.api_key if config.api_key else "cant-be-empty"
+    )
 
-    # TODO: make async
-    def whisper_handler(
+    async def whisper_handler(
         file_path: str, model: str, task: Task, temperature: float, stream: bool
-    ) -> Generator[str, None, None]:
+    ) -> AsyncGenerator[str, None]:
         if task == Task.TRANSCRIBE:
             endpoint = TRANSCRIPTION_ENDPOINT
         elif task == Task.TRANSLATE:

 
         if stream:
             previous_transcription = ""
-            for transcription in streaming_audio_task(file_path, endpoint, temperature, model):
+            async for transcription in streaming_audio_task(file_path, endpoint, temperature, model):
                 previous_transcription += transcription
                 yield previous_transcription
         else:
-            yield audio_task(file_path, endpoint, temperature, model)
+            yield await audio_task(file_path, endpoint, temperature, model)
 
-    def audio_task(file_path: str, endpoint: str, temperature: float, model: str) -> str:
-        with Path(file_path).open("rb") as file:
-            response = http_client.post(
+    async def audio_task(file_path: str, endpoint: str, temperature: float, model: str) -> str:
+        with Path(file_path).open("rb") as file:  # noqa: ASYNC230
+            response = await http_client.post(
                 endpoint,
                 files={"file": file},
                 data={

         response.raise_for_status()
         return response.text
 
-    def streaming_audio_task(
+    async def streaming_audio_task(
         file_path: str, endpoint: str, temperature: float, model: str
-    ) -> Generator[str, None, None]:
-        with Path(file_path).open("rb") as file:
+    ) -> AsyncGenerator[str, None]:
+        with Path(file_path).open("rb") as file:  # noqa: ASYNC230
             kwargs = {
                 "files": {"file": file},
                 "data": {

                     "stream": True,
                 },
             }
-            with connect_sse(http_client, "POST", endpoint, **kwargs) as event_source:
-                for event in event_source.iter_sse():
+            async with aconnect_sse(http_client, "POST", endpoint, **kwargs) as event_source:
+                async for event in event_source.aiter_sse():
                     yield event.data
 
-    def update_whisper_model_dropdown() -> gr.Dropdown:
-        models = openai_client.models.list().data
+    async def update_whisper_model_dropdown() -> gr.Dropdown:
+        models = (await openai_client.models.list()).data
         model_names: list[str] = [model.id for model in models]
         assert config.whisper.model in model_names
         recommended_models = {model for model in model_names if model.startswith("Systran")}

             value=config.whisper.model,
         )
 
-    def update_piper_voices_dropdown() -> gr.Dropdown:
-        res = http_client.get("/v1/audio/speech/voices").raise_for_status()
+    async def update_piper_voices_dropdown() -> gr.Dropdown:
+        res = (await http_client.get("/v1/audio/speech/voices")).raise_for_status()
         piper_models = [PiperModel.model_validate(x) for x in res.json()]
         return gr.Dropdown(choices=[model.voice for model in piper_models], label="Voice", value=DEFAULT_VOICE)
 
-    # TODO: make async
-    def handle_audio_speech(text: str, voice: str, response_format: str, speed: float, sample_rate: int | None) -> Path:
-        res = openai_client.audio.speech.create(
+    async def handle_audio_speech(
+        text: str, voice: str, response_format: str, speed: float, sample_rate: int | None
+    ) -> Path:
+        res = await openai_client.audio.speech.create(
             input=text,
             model="piper",
             voice=voice,  # pyright: ignore[reportArgumentType]

         )
         audio_bytes = res.response.read()
         file_path = Path(f"audio.{response_format}")
-        with file_path.open("wb") as file:
+        with file_path.open("wb") as file:  # noqa: ASYNC230
             file.write(audio_bytes)
         return file_path
 

Add a comment

Open 0
Closed 0

List

...	...	@@ -1,10 +1,10 @@
1		-from collections.abc import Generator
	1	+from collections.abc import AsyncGenerator
2	2	from pathlib import Path
3	3
4	4	import gradio as gr
5	5	import httpx
6		-from httpx_sse import connect_sse
7		-from openai import OpenAI
	6	+from httpx_sse import aconnect_sse
	7	+from openai import AsyncOpenAI
8	8
9	9	from faster_whisper_server.config import Config, Task
10	10	from faster_whisper_server.hf_utils import PiperModel
...	...	@@ -25,13 +25,19 @@
25	25
26	26	def create_gradio_demo(config: Config) -> gr.Blocks: # noqa: C901, PLR0915
27	27	base_url = f"http://{config.host}:{config.port}"
28		- http_client = httpx.Client(base_url=base_url, timeout=TIMEOUT)
29		- openai_client = OpenAI(base_url=f"{base_url}/v1", api_key="cant-be-empty")
	28	+ # TODO: test that auth works
	29	+ http_client = httpx.AsyncClient(
	30	+ base_url=base_url,
	31	+ timeout=TIMEOUT,
	32	+ headers={"Authorization": f"Bearer {config.api_key}"} if config.api_key else {},
	33	+ )
	34	+ openai_client = AsyncOpenAI(
	35	+ base_url=f"{base_url}/v1", api_key=config.api_key if config.api_key else "cant-be-empty"
	36	+ )
30	37
31		- # TODO: make async
32		- def whisper_handler(
	38	+ async def whisper_handler(
33	39	file_path: str, model: str, task: Task, temperature: float, stream: bool
34		- ) -> Generator[str, None, None]:
	40	+ ) -> AsyncGenerator[str, None]:
35	41	if task == Task.TRANSCRIBE:
36	42	endpoint = TRANSCRIPTION_ENDPOINT
37	43	elif task == Task.TRANSLATE:
...	...	@@ -39,15 +45,15 @@
39	45
40	46	if stream:
41	47	previous_transcription = ""
42		- for transcription in streaming_audio_task(file_path, endpoint, temperature, model):
	48	+ async for transcription in streaming_audio_task(file_path, endpoint, temperature, model):
43	49	previous_transcription += transcription
44	50	yield previous_transcription
45	51	else:
46		- yield audio_task(file_path, endpoint, temperature, model)
	52	+ yield await audio_task(file_path, endpoint, temperature, model)
47	53
48		- def audio_task(file_path: str, endpoint: str, temperature: float, model: str) -> str:
49		- with Path(file_path).open("rb") as file:
50		- response = http_client.post(
	54	+ async def audio_task(file_path: str, endpoint: str, temperature: float, model: str) -> str:
	55	+ with Path(file_path).open("rb") as file: # noqa: ASYNC230
	56	+ response = await http_client.post(
51	57	endpoint,
52	58	files={"file": file},
53	59	data={
...	...	@@ -60,10 +66,10 @@
60	66	response.raise_for_status()
61	67	return response.text
62	68
63		- def streaming_audio_task(
	69	+ async def streaming_audio_task(
64	70	file_path: str, endpoint: str, temperature: float, model: str
65		- ) -> Generator[str, None, None]:
66		- with Path(file_path).open("rb") as file:
	71	+ ) -> AsyncGenerator[str, None]:
	72	+ with Path(file_path).open("rb") as file: # noqa: ASYNC230
67	73	kwargs = {
68	74	"files": {"file": file},
69	75	"data": {
...	...	@@ -73,12 +79,12 @@
73	79	"stream": True,
74	80	},
75	81	}
76		- with connect_sse(http_client, "POST", endpoint, **kwargs) as event_source:
77		- for event in event_source.iter_sse():
	82	+ async with aconnect_sse(http_client, "POST", endpoint, **kwargs) as event_source:
	83	+ async for event in event_source.aiter_sse():
78	84	yield event.data
79	85
80		- def update_whisper_model_dropdown() -> gr.Dropdown:
81		- models = openai_client.models.list().data
	86	+ async def update_whisper_model_dropdown() -> gr.Dropdown:
	87	+ models = (await openai_client.models.list()).data
82	88	model_names: list[str] = [model.id for model in models]
83	89	assert config.whisper.model in model_names
84	90	recommended_models = {model for model in model_names if model.startswith("Systran")}
...	...	@@ -90,14 +96,15 @@
90	96	value=config.whisper.model,
91	97	)
92	98
93		- def update_piper_voices_dropdown() -> gr.Dropdown:
94		- res = http_client.get("/v1/audio/speech/voices").raise_for_status()
	99	+ async def update_piper_voices_dropdown() -> gr.Dropdown:
	100	+ res = (await http_client.get("/v1/audio/speech/voices")).raise_for_status()
95	101	piper_models = [PiperModel.model_validate(x) for x in res.json()]
96	102	return gr.Dropdown(choices=[model.voice for model in piper_models], label="Voice", value=DEFAULT_VOICE)
97	103
98		- # TODO: make async
99		- def handle_audio_speech(text: str, voice: str, response_format: str, speed: float, sample_rate: int \| None) -> Path:
100		- res = openai_client.audio.speech.create(
	104	+ async def handle_audio_speech(
	105	+ text: str, voice: str, response_format: str, speed: float, sample_rate: int \| None
	106	+ ) -> Path:
	107	+ res = await openai_client.audio.speech.create(
101	108	input=text,
102	109	model="piper",
103	110	voice=voice, # pyright: ignore[reportArgumentType]
...	...	@@ -107,7 +114,7 @@
107	114	)
108	115	audio_bytes = res.response.read()
109	116	file_path = Path(f"audio.{response_format}")
110		- with file_path.open("wb") as file:
	117	+ with file_path.open("wb") as file: # noqa: ASYNC230
111	118	file.write(audio_bytes)
112	119	return file_path
113	120

Delete comment