Update app.py

app.py

@@ -1,16 +1,17 @@
 import streamlit as st
-import pyttsx3
+from transformers import pipeline
+import numpy as np
 import threading
-import wave
-import io
-import speech_recognition as sr
 from gradio_client import Client
-import streamlit.components.v1 as components
+from streamlit_audio_recorder import st_audiorec
 
-# Initialize session state
+# Initialize session state for chat history
 if "messages" not in st.session_state:
     st.session_state["messages"] = []  # Store chat history
 
+# Load the ASR model using the Hugging Face transformers pipeline
+transcriber = pipeline("automatic-speech-recognition", model="openai/whisper-base.en")
+
 # Function to generate a response using Gradio client
 def generate_response(query):
     try:
@@ -35,6 +36,7 @@ def handle_user_input(user_input):
 
 # Function to speak text (Voice Output)
 def speak_text(text):
+    import pyttsx3
     engine = pyttsx3.init()
     engine.stop()  # Ensure no previous loop is running
     engine.say(text)
@@ -49,69 +51,15 @@ def update_chat_history():
         if "bot" in msg:
             st.markdown(f"<div class='chat-bubble bot-message'><strong>Bot:</strong> {msg['bot']}</div>", unsafe_allow_html=True)
 
-# Function to recognize speech from audio received as bytes
-def recognize_speech_from_audio(audio_bytes):
-    st.info("Processing audio...")
-
-    # Convert byte stream to audio file
-    audio_data = io.BytesIO(audio_bytes)
-    recognizer = sr.Recognizer()
+# Function to process and transcribe audio
+def transcribe_audio(audio_data, sr):
+    # Normalize audio to float32
+    audio_data = audio_data.astype(np.float32)
+    audio_data /= np.max(np.abs(audio_data))
 
-    # Recognize speech from the audio data
-    with sr.AudioFile(audio_data) as source:
-        audio = recognizer.record(source)
-
-    try:
-        recognized_text = recognizer.recognize_google(audio)
-        st.session_state["user_input"] = recognized_text
-        st.success(f"Recognized Text: {recognized_text}")
-        handle_user_input(recognized_text)
-    except sr.UnknownValueError:
-        st.error("Sorry, I couldn't understand the audio.")
-    except sr.RequestError:
-        st.error("Could not request results; please check your internet connection.")
-
-# JavaScript for audio recording and sending data to Streamlit
-audio_recorder_html = """
-    <script>
-        let audioChunks = [];
-        let mediaRecorder;
-        
-        function startRecording() {
-            navigator.mediaDevices.getUserMedia({ audio: true })
-                .then(function(stream) {
-                    mediaRecorder = new MediaRecorder(stream);
-                    mediaRecorder.ondataavailable = function(event) {
-                        audioChunks.push(event.data);
-                    };
-                    mediaRecorder.onstop = function() {
-                        const audioBlob = new Blob(audioChunks, { type: 'audio/wav' });
-                        const reader = new FileReader();
-                        reader.onloadend = function() {
-                            const audioBase64 = reader.result.split(',')[1];
-                            window.parent.postMessage({ 'type': 'audio_data', 'audio': audioBase64 }, '*');
-                        };
-                        reader.readAsDataURL(audioBlob);
-                    };
-                    mediaRecorder.start();
-                });
-        }
-        
-        function stopRecording() {
-            mediaRecorder.stop();
-        }
-        
-        function handleStartStop() {
-            if (mediaRecorder && mediaRecorder.state === "recording") {
-                stopRecording();
-            } else {
-                startRecording();
-            }
-        }
-    </script>
-    <button onclick="handleStartStop()">Start/Stop Recording</button>
-    <p>Click the button to start/stop audio recording.</p>
-"""
+    # Use the ASR model to transcribe the audio
+    transcription = transcriber({"sampling_rate": sr, "raw": audio_data})["text"]
+    return transcription
 
 # Main Streamlit app
 st.set_page_config(page_title="Llama2 Chatbot", page_icon="🤖", layout="wide")
@@ -170,7 +118,7 @@ st.markdown(
     """
     Welcome to the *Llama2 Chatbot*!   
     - *Type* your message below, or   
-    - *Speak* to the bot using your microphone.   
+    - *Use the microphone* to speak to the bot.   
     """
 )
 
@@ -186,17 +134,25 @@ with chat_history_container:
         if submit_button:
             handle_user_input(user_input)
 
-    # Display JavaScript for audio recording
-    components.html(audio_recorder_html, height=300)
+    # Separate button for speech recognition outside of the form
+    if st.button("Speak"):
+        # Record and process the speech using Streamlit Audio Recorder
+        audio_data, sr = st_audiorec()
+        
+        if audio_data is not None:
+            st.audio(audio_data, format="audio/wav")
 
-    # Update chat history on every interaction
-    update_chat_history()
+            # Convert to numpy array
+            audio_np = np.array(audio_data)
 
-# Listening to the audio data sent by JavaScript
-def process_audio_data():
-    audio_data = st.experimental_get_query_params().get('audio', [None])[0]
-    if audio_data:
-        recognize_speech_from_audio(audio_data)
+            # Transcribe the audio
+            transcription = transcribe_audio(audio_np, sr)
 
-# Call the function to process audio if available
-process_audio_data()
+            # Display the recognized text
+            st.session_state["user_input"] = transcription
+            st.success(f"Recognized Text: {transcription}")
+            handle_user_input(transcription)
+
+    st.markdown("### Chat History")
+    # Update chat history on every interaction
+    update_chat_history()