Gradio changes for voice support

app.py

@@ -1,50 +1,67 @@
-import traceback
+import gradio as gr
+from main import index, run
+from gtts import gTTS
+import os
 
-from flask import Flask, render_template
-from flask_cors import CORS
-from flask_executor import Executor
-from flask_socketio import SocketIO, emit
-from gevent import monkey
-from utils import get_search_index
+from transformers import pipeline
 
-from main import run
+p = pipeline("automatic-speech-recognition")
 
-monkey.patch_all(ssl=False)
+"""Use text to call chat method from main.py"""
+def add_text(history, text):
+    print("Question asked: " + text)
+    response = run_model(text)
+    history = history + [(text, response)]
+    print(history)
+    return history, ""
 
-app = Flask(__name__)
-app.config['SECRET_KEY'] = 'secret!'
 
-socketio = SocketIO(app, cors_allowed_origins="*", async_mode='gevent', logger=True)
-cors = CORS(app)
-executor = Executor(app)
+def run_model(text):
+    response = run(question=text)
+    # If response contains string `SOURCES:`, then add a \n before `SOURCES`
+    if "SOURCES:" in response:
+        response = response.replace("SOURCES:", "\nSOURCES:")
+    print(response)
+    return response
 
-executor.init_app(app)
-app.config['EXECUTOR_MAX_WORKERS'] = 5
 
-@app.route('/')
-def index():
-    get_search_index()
-    return render_template('index.html')
 
+def fill_input_textbox(history, audio):
 
-@socketio.on('message')
-def handle_message(data):
-    question = data['question']
-    print("question: " + question)
+    txt = p(audio)["text"]
+    response = run_model(txt)
+    # Remove all text from SOURCES: to the end of the string
+    trimmed_response = response.split("SOURCES:")[0]
+    myobj = gTTS(text=trimmed_response, lang='en', slow=False)
+    myobj.save("response.wav")
+    history = history + [((audio, ), ('response.wav', ))]
+    print(history)
+    return history
 
-    if executor.futures:
-        emit('response', {'response': 'Server is busy, please try again later'})
-        return
+def bot(history):
+    return history
 
-    try:
-        future = executor.submit(run, question)
-        response = future.result()
-        emit('response', {'response': response})
-    except Exception as e:
-        traceback.print_exc()
-        # print(f"Error processing request: {str(e)}")
-        emit('response', {'response': 'Server is busy. Please try again later.'})
+with gr.Blocks() as demo:
+    index()
+    chatbot = gr.Chatbot([(None,'Hi I am Coursera Bot for 3D printing Evolution')], elem_id="chatbot").style(height=750)
 
+    with gr.Row():
+        with gr.Column(scale=0.85):
+            txt = gr.Textbox(
+                label="Coursera Voice Q&A Bot",
+                placeholder="Enter text and press enter, or upload an image", lines=1
+            ).style(container=False)
 
-if __name__ == '__main__':
-    socketio.run(app, host="0.0.0.0", port=7860)
+        with gr.Column(scale=0.15):
+            audio = gr.Audio(source="microphone", type="filepath").style(container=False)
+
+    txt.submit(add_text, [chatbot, txt], [chatbot, txt], postprocess=False).then(
+        bot, chatbot, chatbot
+    )
+
+    audio.change(fn=fill_input_textbox,inputs=[chatbot, audio], outputs=[chatbot]).then(
+        bot, chatbot, chatbot
+    )
+
+if __name__ == "__main__":
+    demo.launch()

app_flask.py

@@ -0,0 +1,71 @@
+import os.path
+import traceback
+
+from flask import Flask, render_template, request
+from flask_cors import CORS
+from flask_executor import Executor
+from flask_socketio import SocketIO, emit
+from gevent import monkey
+from utils import get_search_index
+from scipy.io import wavfile
+import base64, io
+import numpy as np
+import whisper
+from main import run
+
+monkey.patch_all(ssl=False)
+
+app = Flask(__name__)
+app.config['SECRET_KEY'] = 'secret!'
+
+socketio = SocketIO(app, cors_allowed_origins="*", logger=True)
+# cors = CORS(app)
+executor = Executor(app)
+
+executor.init_app(app)
+app.config['EXECUTOR_MAX_WORKERS'] = 5
+
+model = whisper.load_model('small.en')
+
+@app.route('/')
+def index():
+    get_search_index()
+    return render_template('index.html')
+
+
+@socketio.on('message')
+def handle_message(data):
+    question = data['question']
+    print("question: " + question)
+
+    if executor.futures:
+        emit('response', {'response': 'Server is busy, please try again later'})
+        return
+
+    try:
+        future = executor.submit(run, question)
+        response = future.result()
+        emit('response', {'response': response})
+    except Exception as e:
+        traceback.print_exc()
+        # print(f"Error processing request: {str(e)}")
+        emit('response', {'response': 'Server is busy. Please try again later.'})
+
+@app.route('/audio', methods=['POST'])
+def handle_audio():
+    # print the request files and names
+    print(request.files)
+    audio_data = request.files['audio']
+    audio_data.save('audio.webm')
+    print("audio data received: " + str(audio_data))
+
+    if os.path.isfile('audio.webm'):
+        print("audio file exists")
+        # Transcribe the audio data using OpenAI Whisper
+        transcript = whisper.transcribe(model, 'audio.webm')
+        data = {'question': transcript['text']}
+        handle_message(data)
+
+
+if __name__ == '__main__':
+    socketio.run(app, port=5001)

main.py

@@ -4,7 +4,7 @@ question_starters = ['who', 'why', 'what', 'how', 'where', 'when', 'which', 'who
 
 
 def index():
-    create_index()
+    get_search_index()
     return True
 
 def run(question):

requirements.txt

@@ -7,4 +7,8 @@ flask-executor==1.0.0
 gevent==22.10.2
 gevent-websocket==0.10.1
 unstructured==0.5.8
-flask-cors==3.0.10
+flask-cors==3.0.10
+gradio
+ffmpeg-python
+transformers
+gtts