Spaces:
Runtime error
Runtime error
refactor: rename webrtc... to app_interface...
Browse files
webrtc_av_queue_actor.py → app_interface_actor.py
RENAMED
@@ -6,13 +6,19 @@ import numpy as np
|
|
6 |
|
7 |
|
8 |
@ray.remote
|
9 |
-
class
|
10 |
def __init__(self):
|
11 |
self.audio_input_queue = Queue(maxsize=3000) # Adjust the size as needed
|
12 |
self.video_input_queue = Queue(maxsize=10) # Adjust the size as needed
|
13 |
self.audio_output_queue = Queue(maxsize=3000) # Adjust the size as needed
|
14 |
self.video_output_queue = Queue(maxsize=10) # Adjust the size as needed
|
15 |
|
|
|
|
|
|
|
|
|
|
|
|
|
16 |
|
17 |
async def enqueue_video_input_frame(self, shared_tensor_ref):
|
18 |
if self.video_input_queue.full():
|
|
|
6 |
|
7 |
|
8 |
@ray.remote
|
9 |
+
class AppInterfaceActor:
|
10 |
def __init__(self):
|
11 |
self.audio_input_queue = Queue(maxsize=3000) # Adjust the size as needed
|
12 |
self.video_input_queue = Queue(maxsize=10) # Adjust the size as needed
|
13 |
self.audio_output_queue = Queue(maxsize=3000) # Adjust the size as needed
|
14 |
self.video_output_queue = Queue(maxsize=10) # Adjust the size as needed
|
15 |
|
16 |
+
@staticmethod
|
17 |
+
def get_singleton():
|
18 |
+
return AppInterfaceActor.options(
|
19 |
+
name="AppInterfaceActor",
|
20 |
+
get_if_exists=True,
|
21 |
+
).remote()
|
22 |
|
23 |
async def enqueue_video_input_frame(self, shared_tensor_ref):
|
24 |
if self.video_input_queue.full():
|
streamlit_av_queue.py
CHANGED
@@ -8,7 +8,7 @@ import cv2
|
|
8 |
import numpy as np
|
9 |
import ray
|
10 |
from ray.util.queue import Queue
|
11 |
-
from
|
12 |
import pydub
|
13 |
import torch
|
14 |
|
@@ -19,10 +19,7 @@ class StreamlitAVQueue:
|
|
19 |
self._listening = True
|
20 |
self._looking = False
|
21 |
self._lock = threading.Lock()
|
22 |
-
self.
|
23 |
-
name="WebRtcAVQueueActor",
|
24 |
-
get_if_exists=True,
|
25 |
-
).remote()
|
26 |
self._video_output_frame = None
|
27 |
|
28 |
def set_looking_listening(self, looking, listening: bool):
|
@@ -38,14 +35,14 @@ class StreamlitAVQueue:
|
|
38 |
try:
|
39 |
with self._lock:
|
40 |
should_look = self._looking
|
41 |
-
next_video_output_frame = await self.
|
42 |
if next_video_output_frame is not None:
|
43 |
self._video_output_frame = next_video_output_frame
|
44 |
for i, frame in enumerate(frames):
|
45 |
user_image = frame.to_ndarray(format="rgb24")
|
46 |
if should_look:
|
47 |
shared_tensor_ref = ray.put(user_image)
|
48 |
-
await self.
|
49 |
if self._video_output_frame is not None:
|
50 |
frame = self._video_output_frame
|
51 |
# resize user image to 1/4 size
|
@@ -85,7 +82,7 @@ class StreamlitAVQueue:
|
|
85 |
sound_chunk += sound
|
86 |
shared_buffer = np.array(sound_chunk.get_array_of_samples())
|
87 |
shared_buffer_ref = ray.put(shared_buffer)
|
88 |
-
await self.
|
89 |
except Exception as e:
|
90 |
print (e)
|
91 |
|
@@ -97,7 +94,7 @@ class StreamlitAVQueue:
|
|
97 |
# print (f"frame: {frame.format.name}, {frame.layout.name}, {frame.sample_rate}, {frame.samples}")
|
98 |
assert frame.format.bytes == 2
|
99 |
assert frame.format.name == 's16'
|
100 |
-
frame_as_bytes = await self.
|
101 |
if frame_as_bytes:
|
102 |
# print(f"frame_as_bytes: {len(frame_as_bytes)}")
|
103 |
assert len(frame_as_bytes) == frame.samples * frame.format.bytes
|
@@ -116,15 +113,15 @@ class StreamlitAVQueue:
|
|
116 |
return new_frames
|
117 |
|
118 |
async def get_audio_input_frames_async(self) -> List[av.AudioFrame]:
|
119 |
-
shared_buffers = await self.
|
120 |
return shared_buffers
|
121 |
|
122 |
async def get_video_frames_async(self) -> List[av.AudioFrame]:
|
123 |
-
shared_tensors = await self.
|
124 |
return shared_tensors
|
125 |
|
126 |
def get_audio_output_queue(self)->Queue:
|
127 |
-
return self.
|
128 |
|
129 |
def get_video_output_queue(self)->Queue:
|
130 |
-
return self.
|
|
|
8 |
import numpy as np
|
9 |
import ray
|
10 |
from ray.util.queue import Queue
|
11 |
+
from app_interface_actor import AppInterfaceActor
|
12 |
import pydub
|
13 |
import torch
|
14 |
|
|
|
19 |
self._listening = True
|
20 |
self._looking = False
|
21 |
self._lock = threading.Lock()
|
22 |
+
self.app_interface_actor = AppInterfaceActor.get_singleton()
|
|
|
|
|
|
|
23 |
self._video_output_frame = None
|
24 |
|
25 |
def set_looking_listening(self, looking, listening: bool):
|
|
|
35 |
try:
|
36 |
with self._lock:
|
37 |
should_look = self._looking
|
38 |
+
next_video_output_frame = await self.app_interface_actor.get_video_output_frame.remote()
|
39 |
if next_video_output_frame is not None:
|
40 |
self._video_output_frame = next_video_output_frame
|
41 |
for i, frame in enumerate(frames):
|
42 |
user_image = frame.to_ndarray(format="rgb24")
|
43 |
if should_look:
|
44 |
shared_tensor_ref = ray.put(user_image)
|
45 |
+
await self.app_interface_actor.enqueue_video_input_frame.remote(shared_tensor_ref)
|
46 |
if self._video_output_frame is not None:
|
47 |
frame = self._video_output_frame
|
48 |
# resize user image to 1/4 size
|
|
|
82 |
sound_chunk += sound
|
83 |
shared_buffer = np.array(sound_chunk.get_array_of_samples())
|
84 |
shared_buffer_ref = ray.put(shared_buffer)
|
85 |
+
await self.app_interface_actor.enqueue_audio_input_frame.remote(shared_buffer_ref)
|
86 |
except Exception as e:
|
87 |
print (e)
|
88 |
|
|
|
94 |
# print (f"frame: {frame.format.name}, {frame.layout.name}, {frame.sample_rate}, {frame.samples}")
|
95 |
assert frame.format.bytes == 2
|
96 |
assert frame.format.name == 's16'
|
97 |
+
frame_as_bytes = await self.app_interface_actor.get_audio_output_frame.remote()
|
98 |
if frame_as_bytes:
|
99 |
# print(f"frame_as_bytes: {len(frame_as_bytes)}")
|
100 |
assert len(frame_as_bytes) == frame.samples * frame.format.bytes
|
|
|
113 |
return new_frames
|
114 |
|
115 |
async def get_audio_input_frames_async(self) -> List[av.AudioFrame]:
|
116 |
+
shared_buffers = await self.app_interface_actor.get_audio_input_frames.remote()
|
117 |
return shared_buffers
|
118 |
|
119 |
async def get_video_frames_async(self) -> List[av.AudioFrame]:
|
120 |
+
shared_tensors = await self.app_interface_actor.get_video_input_frames.remote()
|
121 |
return shared_tensors
|
122 |
|
123 |
def get_audio_output_queue(self)->Queue:
|
124 |
+
return self.app_interface_actor.get_audio_output_queue.remote()
|
125 |
|
126 |
def get_video_output_queue(self)->Queue:
|
127 |
+
return self.app_interface_actor.get_video_output_queue.remote()
|