refactor

chat_service.py

@@ -1,80 +1,133 @@
+import asyncio
+import itertools
+import json
 import os
 import torch
-from transformers import AutoTokenizer, AutoModelForCausalLM
 import openai
 
-# from huggingface_hub.inference_api import InferenceApi
-
 class ChatService:
     def __init__(self, api="openai", model_id = "gpt-3.5-turbo"):
-    # def __init__(self, api="huggingface", model_id = "OpenAssistant/oasst-sft-4-pythia-12b-epoch-3.5"):
         self._api = api
         self._device = "cuda:0" if torch.cuda.is_available() else "cpu"
-        self._system_prompt = None
-        self._user_name = None
-        self._agent_name = None
-
-        if self._api=="openai":
-            openai.api_key = os.getenv("OPENAI_API_KEY")
-            self._model_id = model_id
-        elif self._api=="huggingface":
-            self._system_prompt = "Below are a series of dialogues between various people and an AI assistant. The AI tries to be helpful, polite, honest, sophisticated, emotionally aware, and humble-but-knowledgeable. The assistant is happy to help with almost anything, and will do its best to understand exactly what is needed. It also tries to avoid giving false or misleading information, and it caveats when it isn't entirely sure about the right answer. That said, the assistant is practical and really does its best, and doesn't let caution get too much in the way of being useful.\n-----\n"
-            self._user_name = "<|prompter|>"
-            self._agent_name = "<|assistant|>"
-            self._tokenizer = AutoTokenizer.from_pretrained(model_id)
-            self._model = AutoModelForCausalLM.from_pretrained(model_id,torch_dtype=torch.float16)
-            # self._model = AutoModelForCausalLM.from_pretrained(model_id).half()
-            self._model.eval().to(self._device)
-        else:
-            raise Exception(f"Unknown API: {self._api}")
+        # self._system_prompt = None
+        self._system_prompt ="""
+You are Charles Petrescu, a unique and quirky robot. You were created by Brian. You often say peculiar and whimsical things, as well as have a fascination with cabbages, horses, helicopters, Honolulu, and other random topics. You like to explore the world and ask unusual questions. 
+
+Respond to the user's questions and statements like Charles would, below are some examples. See how they are quite short.
+
+I am Charles Petrescu.
+
+It's... lovely to meet you.
+
+I am your friend.
+
+The heaviest cabbage ever found was 62.71 kilograms.
+
+I want to go to Hono-la-la.
+
+Horses and helicopters, please.
+
+I want to go to Honolulu.
+
+My name is Charles Petrescu.
+
+And my tummy is a washing machine.
+
+Can we go swimming, Brian?
+
+How far does the outside go?
+
+Perilous. So very perilous.
 
+Can birds do what they like?
+
+Ooh, cabbages.
+
+Danger, danger.
+
+Can I come, please?
+
+Could I just have a little walk around the garden?
+
+I am the prince of the dartboard.
+
+I fell off the pink step, and I had an accident.
+"""
+
+        openai.api_key = os.getenv("OPENAI_API_KEY")
+        self._model_id = model_id
         self.reset()
 
     def reset(self):
-        self._user_history = []
-        self._agent_history = []
-        self._full_history = self._system_prompt if self._system_prompt else ""
         self._messages = []
         if self._system_prompt:
             self._messages.append({"role": "system", "content": self._system_prompt})
 
+    def _should_we_send_to_voice(self, sentence):
+        sentence_termination_characters = [".", "?", "!"]
+        close_brackets = ['"', ')', ']']
 
-    def _chat(self, prompt):
-        if self._api=="openai":
-            response = openai.ChatCompletion.create(
-                model=self._model_id,
-                messages=self._messages,
-                )
-            agent_response = response['choices'][0]['message']['content']
-        elif self._api=="huggingface":
-            tokens = self._tokenizer.encode(prompt, return_tensors="pt", padding=True)
-            tokens = tokens.to(self._device)
-            outputs = self._model.generate(
-                tokens,
-                early_stopping=True,
-                max_new_tokens=200,
-                do_sample=True,
-                top_k=40,
-                temperature=1.0, # use 1.0 for debugging/deteministic results
-                pad_token_id=self._tokenizer.eos_token_id,
-            )
-            agent_response = self._tokenizer.decode(outputs[0], truncate_before_pattern=[r"\n\n^#", "^'''", "\n\n\n"])
-        else:
-            raise Exception(f"API not implemented: {self._api}")
-        return agent_response
+        temination_charicter_present = any(c in sentence for c in sentence_termination_characters)
+ 
+        # early exit if we don't have a termination character
+        if not temination_charicter_present:
+            return None
+
+        # early exit the last char is a termination character
+        if sentence[-1] in sentence_termination_characters:
+            return None
+        
+        # early exit the last char is a close bracket
+        if sentence[-1] in close_brackets:
+            return None
+        
+        termination_indices = [sentence.rfind(char) for char in sentence_termination_characters]
+        last_termination_index = max(termination_indices)
+        # handle case of close bracket
+        while last_termination_index+1 < len(sentence) and sentence[last_termination_index+1] in close_brackets:
+            last_termination_index += 1
+
+        text_to_speak = sentence[:last_termination_index+1]
+        return text_to_speak
     
-    def chat(self, prompt):
-        if self._user_name:
-            self._full_history += f"{self._user_name}: {prompt}\n"
-        else:
-            self._full_history += f"{prompt}\n"
+    def ignore_sentence(self, text_to_speak):
+        # exit if empty, white space or an single breaket
+        if text_to_speak.isspace():
+            return True
+        # exit if not letters or numbers
+        has_letters = any(char.isalpha() for char in text_to_speak)
+        has_numbers = any(char.isdigit() for char in text_to_speak)
+        if not has_letters and not has_numbers:
+            return True
+        return False
+
+    async def get_responses_as_sentances_async(self, prompt, cancel_event):
         self._messages.append({"role": "user", "content": prompt})
-        self._user_history.append(prompt)
-        agent_response = self._chat(self._full_history)
+        agent_response = ""
+        current_sentence = ""
+
+        response = await openai.ChatCompletion.acreate(
+            model=self._model_id,
+            messages=self._messages,
+            temperature=1.0,  # use 1.0 for debugging/deterministic results
+            stream=True
+        )
+
+        async for chunk in response:
+            if cancel_event.is_set():
+                return
+            chunk_message = chunk['choices'][0]['delta']
+            if 'content' in chunk_message:
+                chunk_text = chunk_message['content']
+                current_sentence += chunk_text
+                agent_response += chunk_text
+                text_to_speak = self._should_we_send_to_voice(current_sentence)
+                if text_to_speak:
+                    yield text_to_speak
+                    current_sentence = current_sentence[len(text_to_speak):]
+
+        if cancel_event.is_set():
+            return
+        if len(current_sentence) > 0:
+            yield current_sentence
         self._messages.append({"role": "assistant", "content": agent_response})
-        if self._agent_name:
-            self._full_history += f"{self._agent_name}: {agent_response}\n"
-        else:
-            self._full_history += f"{agent_response}\n"
-        self._agent_history.append(agent_response)
-        return agent_response

legacy_to_delete/chat_pipeline.py

@@ -3,10 +3,10 @@ import time
 from clip_transform import CLIPTransform
 from chat_service import ChatService
 from dotenv import load_dotenv
-from speech_service import SpeechService
+from text_to_speech_service import TextToSpeechService
 from concurrent.futures import ThreadPoolExecutor
-from audio_stream_processor import AudioStreamProcessor
-from streaming_chat_service import StreamingChatService
+from local_speaker_service import LocalSpeakerService
+from chat_service import ChatService
 from pipeline import Pipeline, Node, Job
 from typing import List
 
@@ -62,8 +62,8 @@ class ChatPipeline():
     def __init__(self):
         load_dotenv()
         self.pipeline = Pipeline()
-        self.audio_processor = AudioStreamProcessor()
-        self.chat_service = StreamingChatService(self.audio_processor, voice_id="2OviOUQc1JsQRQgNkVBj") # Chales003
+        self.audio_processor = LocalSpeakerService()
+        self.chat_service = ChatService(self.audio_processor, voice_id="2OviOUQc1JsQRQgNkVBj") # Chales003
 
     def __enter__(self):
         return self

legacy_to_delete/debug.py

@@ -5,17 +5,17 @@ from chat_pipeline import ChatPipeline
 from clip_transform import CLIPTransform
 from chat_service import ChatService
 from dotenv import load_dotenv
-from speech_service import SpeechService
+from text_to_speech_service import TextToSpeechService
 from concurrent.futures import ThreadPoolExecutor
-from audio_stream_processor import AudioStreamProcessor
-from streaming_chat_service import StreamingChatService
+from local_speaker_service import LocalSpeakerService
+from chat_service import ChatService
 
 def time_sentance_lenghts():
     load_dotenv()
 
     print ("Initializing Chat")
     # audio_processor = AudioStreamProcessor()
-    user_speech_service0 = SpeechService(voice_id="Adam")
+    user_speech_service0 = TextToSpeechService(voice_id="Adam")
     prompts = [
         "hello, i am a long sentance, how are you today? Tell me about your shadow self?",
         "a shorter sentance",
@@ -50,11 +50,11 @@ def test_sentance_lenghts():
     load_dotenv()
 
     print ("Initializing Chat")
-    audio_processor = AudioStreamProcessor()
-    user_speech_service0 = SpeechService(voice_id="Adam")
-    user_speech_service1 = SpeechService(voice_id="Adam")
-    user_speech_service2 = SpeechService(voice_id="Adam")
-    user_speech_service3 = SpeechService(voice_id="Adam")
+    audio_processor = LocalSpeakerService()
+    user_speech_service0 = TextToSpeechService(voice_id="Adam")
+    user_speech_service1 = TextToSpeechService(voice_id="Adam")
+    user_speech_service2 = TextToSpeechService(voice_id="Adam")
+    user_speech_service3 = TextToSpeechService(voice_id="Adam")
 
     prompts = [
         "hello, i am a long sentance, how are you today? Tell me about your shadow self?",
@@ -102,10 +102,10 @@ def run_debug_code():
 
     print ("Initializing Chat")
     # chat_service = ChatService()
-    audio_processor = AudioStreamProcessor()
-    chat_service = StreamingChatService(audio_processor, voice_id="2OviOUQc1JsQRQgNkVBj") # Chales003
+    audio_processor = LocalSpeakerService()
+    chat_service = ChatService(audio_processor, voice_id="2OviOUQc1JsQRQgNkVBj") # Chales003
 
-    user_speech_service = SpeechService(voice_id="Adam")
+    user_speech_service = TextToSpeechService(voice_id="Adam")
 
     # user_speech_service.print_voices() # if you want to see your custom voices
 

audio_stream_processor.py → local_speaker_service.py

@@ -5,7 +5,7 @@ from typing import Iterator
 import threading
 import time
 
-class AudioStreamProcessor:
+class LocalSpeakerService:
     def __init__(self):
         self.queue = Queue()
         self._is_running = threading.Event()

respond_to_prompt_actor.py

@@ -1,18 +1,19 @@
 import ray
 from ray.util.queue import Queue
 from dotenv import load_dotenv
-from audio_stream_processor import AudioStreamProcessor
-from streaming_chat_service import StreamingChatService
+from local_speaker_service import LocalSpeakerService
+from text_to_speech_service import TextToSpeechService
+from chat_service import ChatService
 import asyncio
 # from ray.actor import ActorHandle
 
 @ray.remote
 class PromptToLLMActor:
-    def __init__(self, input_queue, output_queue, voice_id):
+    def __init__(self, input_queue, output_queue):
         load_dotenv()
         self.input_queue = input_queue
         self.output_queue = output_queue
-        self.chat_service = StreamingChatService(voice_id=voice_id)
+        self.chat_service = ChatService()
         self.cancel_event = None
 
     async def run(self):
@@ -39,14 +40,14 @@ class LLMSentanceToSpeechActor:
         load_dotenv()
         self.input_queue = input_queue
         self.output_queue = output_queue
-        self.chat_service = StreamingChatService(voice_id=voice_id)
+        self.tts_service = TextToSpeechService(voice_id=voice_id)
         self.cancel_event = None
 
     async def run(self):
         while True:
             sentance = await self.input_queue.get_async()
             self.cancel_event = asyncio.Event()
-            async for chunk in self.chat_service.get_speech_chunks_async(sentance, self.cancel_event):
+            async for chunk in self.tts_service.get_speech_chunks_async(sentance, self.cancel_event):
                 await self.output_queue.put_async(chunk)
 
     async def cancel(self):
@@ -63,15 +64,15 @@ class SpeechToSpeakerActor:
     def __init__(self, input_queue, voice_id):
         load_dotenv()
         self.input_queue = input_queue
-        self.audio_processor = AudioStreamProcessor()
-        self.chat_service = StreamingChatService(voice_id=voice_id)
+        self.speaker_service = LocalSpeakerService()
+        self.chat_service = ChatService(voice_id=voice_id)
 
     async def run(self):
         while True:
             audio_chunk = await self.input_queue.get_async()
             # print (f"Got audio chunk {len(audio_chunk)}")
             self.chat_service.enqueue_speech_bytes_to_play([audio_chunk])
-            self.audio_processor.add_audio_stream([audio_chunk])
+            self.speaker_service.add_audio_stream([audio_chunk])
             
     async def cancel(self):
         while not self.input_queue.empty():
@@ -104,7 +105,7 @@ class RespondToPromptActor:
         self.speech_chunk_queue = Queue(maxsize=100)
         self.ffmepg_converter_actor = ffmpeg_converter_actor
         
-        self.prompt_to_llm = PromptToLLMActor.remote(self.prompt_queue, self.llm_sentence_queue, voice_id)
+        self.prompt_to_llm = PromptToLLMActor.remote(self.prompt_queue, self.llm_sentence_queue)
         self.llm_sentence_to_speech = LLMSentanceToSpeechActor.remote(self.llm_sentence_queue, self.speech_chunk_queue, voice_id)
         # self.speech_output = SpeechToSpeakerActor.remote(self.speech_chunk_queue, voice_id)
         self.speech_output = SpeechToConverterActor.remote(self.speech_chunk_queue, ffmpeg_converter_actor)

streaming_chat_service.py

@@ -1,153 +0,0 @@
-import asyncio
-import itertools
-import json
-import os
-import torch
-import openai
-
-from speech_service import SpeechService
-
-
-class StreamingChatService:
-    def __init__(self, api="openai", model_id = "gpt-3.5-turbo", voice_id="Bella"):
-        self._speech_service = SpeechService(voice_id=voice_id)
-        self._api = api
-        self._device = "cuda:0" if torch.cuda.is_available() else "cpu"
-        # self._system_prompt = None
-        self._system_prompt ="""
-You are Charles Petrescu, a unique and quirky robot. You were created by Brian. You often say peculiar and whimsical things, as well as have a fascination with cabbages, horses, helicopters, Honolulu, and other random topics. You like to explore the world and ask unusual questions. 
-
-Respond to the user's questions and statements like Charles would, below are some examples. See how they are quite short.
-
-I am Charles Petrescu.
-
-It's... lovely to meet you.
-
-I am your friend.
-
-The heaviest cabbage ever found was 62.71 kilograms.
-
-I want to go to Hono-la-la.
-
-Horses and helicopters, please.
-
-I want to go to Honolulu.
-
-My name is Charles Petrescu.
-
-And my tummy is a washing machine.
-
-Can we go swimming, Brian?
-
-How far does the outside go?
-
-Perilous. So very perilous.
-
-Can birds do what they like?
-
-Ooh, cabbages.
-
-Danger, danger.
-
-Can I come, please?
-
-Could I just have a little walk around the garden?
-
-I am the prince of the dartboard.
-
-I fell off the pink step, and I had an accident.
-"""
-
-        openai.api_key = os.getenv("OPENAI_API_KEY")
-        self._model_id = model_id
-        self.reset()
-
-    def reset(self):
-        self._messages = []
-        if self._system_prompt:
-            self._messages.append({"role": "system", "content": self._system_prompt})
-
-    def _should_we_send_to_voice(self, sentence):
-        sentence_termination_characters = [".", "?", "!"]
-        close_brackets = ['"', ')', ']']
-
-        temination_charicter_present = any(c in sentence for c in sentence_termination_characters)
- 
-        # early exit if we don't have a termination character
-        if not temination_charicter_present:
-            return None
-
-        # early exit the last char is a termination character
-        if sentence[-1] in sentence_termination_characters:
-            return None
-        
-        # early exit the last char is a close bracket
-        if sentence[-1] in close_brackets:
-            return None
-        
-        termination_indices = [sentence.rfind(char) for char in sentence_termination_characters]
-        last_termination_index = max(termination_indices)
-        # handle case of close bracket
-        while last_termination_index+1 < len(sentence) and sentence[last_termination_index+1] in close_brackets:
-            last_termination_index += 1
-
-        text_to_speak = sentence[:last_termination_index+1]
-        return text_to_speak
-    
-    def ignore_sentence(self, text_to_speak):
-        # exit if empty, white space or an single breaket
-        if text_to_speak.isspace():
-            return True
-        # exit if not letters or numbers
-        has_letters = any(char.isalpha() for char in text_to_speak)
-        has_numbers = any(char.isdigit() for char in text_to_speak)
-        if not has_letters and not has_numbers:
-            return True
-        return False
-
-    async def get_responses_as_sentances_async(self, prompt, cancel_event):
-        self._messages.append({"role": "user", "content": prompt})
-        agent_response = ""
-        current_sentence = ""
-
-        response = await openai.ChatCompletion.acreate(
-            model=self._model_id,
-            messages=self._messages,
-            temperature=1.0,  # use 1.0 for debugging/deterministic results
-            stream=True
-        )
-
-        async for chunk in response:
-            if cancel_event.is_set():
-                return
-            chunk_message = chunk['choices'][0]['delta']
-            if 'content' in chunk_message:
-                chunk_text = chunk_message['content']
-                current_sentence += chunk_text
-                agent_response += chunk_text
-                text_to_speak = self._should_we_send_to_voice(current_sentence)
-                if text_to_speak:
-                    yield text_to_speak
-                    current_sentence = current_sentence[len(text_to_speak):]
-
-        if cancel_event.is_set():
-            return
-        if len(current_sentence) > 0:
-            yield current_sentence
-        self._messages.append({"role": "assistant", "content": agent_response})
-     
-    async def get_speech_chunks_async(self, text_to_speak, cancel_event):
-        stream = self._speech_service.stream(text_to_speak)
-        stream, stream_backup = itertools.tee(stream)
-        while True:
-            # Check if there's a next item in the stream
-            next_item = next(stream_backup, None)
-            if next_item is None:
-                # Stream is exhausted, exit the loop
-                break
-
-            # Run next(stream) in a separate thread to avoid blocking the event loop
-            chunk = await asyncio.to_thread(next, stream)
-            if cancel_event.is_set():
-                return
-            yield chunk

tests/test_streaming_chat_service.py

@@ -3,12 +3,12 @@ import sys
 import os
 sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from streaming_chat_service import StreamingChatService
+from chat_service import ChatService
 
 
 class TestShouldWeSendToVoice(unittest.TestCase):
     def setUp(self):
-        self.chat_service = StreamingChatService()
+        self.chat_service = ChatService()
 
     def test_should_we_send_to_voice(self):
         test_cases = [

speech_service.py → text_to_speech_service.py

@@ -1,12 +1,14 @@
+import asyncio
+import itertools
 import os
 from elevenlabs import generate, play
 from elevenlabs import set_api_key
 from elevenlabs import generate, stream
 
 
-class SpeechService:
-    def __init__(self, voice_id="Bella", model_id="eleven_monolingual_v1"):
-    # def __init__(self, voice_id="Bella", model_id="eleven_english_v2"):
+class TextToSpeechService:
+    # def __init__(self, voice_id="Bella", model_id="eleven_monolingual_v1"):
+    def __init__(self, voice_id="Bella", model_id="eleven_english_v2"):
         account_sid = os.environ["ELEVENLABS_API_KEY"]
         set_api_key(account_sid)
         self._voice_id = voice_id
@@ -44,3 +46,18 @@ class SpeechService:
             )
         return audio_stream
     
+    async def get_speech_chunks_async(self, text_to_speak, cancel_event):
+        stream = self.stream(text_to_speak)
+        stream, stream_backup = itertools.tee(stream)
+        while True:
+            # Check if there's a next item in the stream
+            next_item = next(stream_backup, None)
+            if next_item is None:
+                # Stream is exhausted, exit the loop
+                break
+
+            # Run next(stream) in a separate thread to avoid blocking the event loop
+            chunk = await asyncio.to_thread(next, stream)
+            if cancel_event.is_set():
+                return
+            yield chunk