Added audio I/O

app.py

@@ -1,6 +1,6 @@
 import gradio as gr
 
-from llm import end_interview, get_problem, send_request
+from llm import end_interview, get_problem, read_last_message, send_request, transcribe_audio
 from options import languages_list, models, topics_list
 
 
@@ -8,14 +8,43 @@ def hide_settings():
     init_acc = gr.Accordion("Settings", open=False)
     start_btn = gr.Button("Generate a problem", interactive=False)
     solution_acc = gr.Accordion("Solution", open=True)
-    return init_acc, start_btn, solution_acc
+    end_btn = gr.Button("Finish the interview", interactive=True)
+    send_btn = gr.Button("Send", interactive=True)
+    audio_input = gr.Audio(
+        label="Record audio",
+        sources=["microphone"],
+        type="filepath",
+        waveform_options={"show_controls": False},
+        interactive=True,
+        editable=False,
+    )
+    chat = [
+        (
+            None,
+            "Welcome to the interview! Please take a moment to read the problem statement. Then you can share you initial thoughts and ask any questions you may have. Good luck!",
+        )
+    ]
+    return init_acc, start_btn, solution_acc, end_btn, send_btn, audio_input, chat
 
 
 def hide_solution():
     solution_acc = gr.Accordion("Solution", open=False)
     end_btn = gr.Button("Finish the interview", interactive=False)
     problem_acc = gr.Accordion("Problem statement", open=False)
-    return solution_acc, end_btn, problem_acc
+    send_btn = gr.Button("Send", interactive=False)
+    audio_input = gr.Audio(
+        label="Record audio",
+        sources=["microphone"],
+        type="filepath",
+        waveform_options={"show_controls": False},
+        interactive=False,
+        editable=False,
+    )
+    return solution_acc, end_btn, problem_acc, send_btn, audio_input
+
+
+def return_none():
+    return None
 
 
 with gr.Blocks() as demo:
@@ -57,12 +86,22 @@ with gr.Blocks() as demo:
                     language_select = gr.Dropdown(
                         label="Select language", choices=languages_list, value="python", container=False, interactive=True
                     )
-                    code = gr.Code(label="Solution", language=language_select.value, lines=20)
-                    message = gr.Textbox(label="Message", lines=1)
-                    # TODO: add voice input and output
+                    code = gr.Code(label="Solution", language=language_select.value, lines=35)
                 with gr.Column(scale=1):
+                    end_btn = gr.Button("Finish the interview", interactive=False)
                     chat = gr.Chatbot(label="Chat history")
-                    end_btn = gr.Button("Finish the interview")
+                    audio_input = gr.Audio(
+                        label="Record audio",
+                        sources=["microphone"],
+                        type="filepath",
+                        waveform_options={"show_controls": False},
+                        interactive=False,
+                        editable=False,
+                    )
+                    audio_output = gr.Audio(label="Play audio", autoplay=True, visible=False)
+                    message = gr.Textbox(label="Message", lines=3)
+                    send_btn = gr.Button("Send", interactive=False)
+
         with gr.Accordion("Feedback", open=True) as feedback_acc:
             feedback = gr.Markdown()
 
@@ -71,16 +110,28 @@ with gr.Blocks() as demo:
         inputs=[requirements, difficulty_select, topic_select, model_select],
         outputs=[description, chat_history],
         scroll_to_output=True,
-    ).then(fn=hide_settings, inputs=None, outputs=[init_acc, start_btn, solution_acc])
+    ).then(fn=hide_settings, inputs=None, outputs=[init_acc, start_btn, solution_acc, end_btn, send_btn, audio_input, chat])
 
-    message.submit(
+    send_btn.click(
         fn=send_request,
         inputs=[code, previous_code, message, chat_history, chat, model_select],
         outputs=[chat_history, chat, message, previous_code],
     )
 
     end_btn.click(fn=end_interview, inputs=[chat_history, model_select], outputs=feedback).then(
-        fn=hide_solution, inputs=None, outputs=[solution_acc, end_btn, problem_acc]
+        fn=hide_solution, inputs=None, outputs=[solution_acc, end_btn, problem_acc, send_btn, audio_input]
     )
 
+    audio_input.stop_recording(fn=transcribe_audio, inputs=[audio_input], outputs=[message]).then(
+        fn=return_none, inputs=None, outputs=[audio_input]
+    ).then(
+        fn=send_request,
+        inputs=[code, previous_code, message, chat_history, chat, model_select],
+        outputs=[chat_history, chat, message, previous_code],
+    )
+
+    chat.change(fn=read_last_message, inputs=[chat], outputs=[audio_output])
+
+    audio_output.stop(fn=return_none, inputs=None, outputs=[audio_output])
+
 demo.launch()

llm.py

@@ -68,7 +68,25 @@ def send_request(code, previous_code, message, chat_history, chat_display, model
         reply = "There was an error processing your request."
 
     chat_history.append({"role": "assistant", "content": json_reply})
-
     chat_display.append([message, str(reply)])
 
     return chat_history, chat_display, "", code
+
+
+def transcribe_audio(filename, client=client):
+    with open(filename, "rb") as audio_file:
+        transcription = client.audio.transcriptions.create(model="whisper-1", file=audio_file, response_format="text")
+
+    return transcription
+
+
+def text_to_speech(text, client=client):
+    response = client.audio.speech.create(model="tts-1", voice="alloy", input=text)
+    return response.content
+
+
+def read_last_message(chat_display):
+    last_message = chat_display[-1][1]
+
+    audio = text_to_speech(last_message)
+    return audio