const startStopButton = document.getElementById('startStopButton'); const voiceSelectionDropdown = document.getElementById('voiceSelect'); const modelSelectionDropdown = document.getElementById('modelSelect'); const noiseSuppressionCheckbox = document.getElementById('noiseSuppression'); const responseTimeDisplay = document.getElementById('responseTime'); const userActivityIndicator = document.getElementById('userIndicator'); const aiActivityIndicator = document.getElementById('aiIndicator'); const transcriptDiv = document.getElementById('transcript'); let speechRecognizer; let activeQuery = null; let queryStartTime = 0; let completeTranscript = ''; let isRequestInProgress = false; let isUserSpeaking = false; let isSpeechRecognitionActive = false; let userManuallyStoppedRecognizer = false; let requestAbortController = null; let partialTranscript = ''; let lastUserSpeechTimestamp = null; let prefetchTextQuery = ""; let firstResponseTextTimestamp = null; // Configuration const USER_SPEECH_INTERRUPT_DELAY = 500; const TEXT_TO_SPEECH_API_ENDPOINT = "https://api.streamelements.com/kappa/v2/speech"; const CHUNK_SIZE = 300; // Audio Management let currentAudio = null; let audioPlaybackQueue = []; let prefetchQueue = []; // Enhanced Prefetching and Caching const prefetchCache = new Map(); const pendingPrefetchRequests = new Map(); const MAX_PREFETCH_REQUESTS = 10; const prefetchCacheExpiration = 60000; // 1 minute // Global Conversation History let conversationHistory = []; // Audio Caching const audioCache = new Map(); const audioCacheExpiration = 3600000; // 1 hour // Normalize query text const normalizeQueryText = query => query.trim().toLowerCase().replace(/[^\w\s]/g, ''); // Generate a cache key const generateCacheKey = (normalizedQuery, voice, history, modelName) => `${normalizedQuery}-${voice}-${JSON.stringify(history)}-${modelName}`; // Prefetch and cache the first TTS audio chunk const prefetchFirstAudioChunk = (query, voice) => { const normalizedQuery = normalizeQueryText(query); const cacheKey = generateCacheKey(normalizedQuery, voice, conversationHistory, modelSelectionDropdown.value); if (pendingPrefetchRequests.has(cacheKey) || prefetchCache.has(cacheKey)) return; prefetchQueue.push({ query:query.trim(), voice, cacheKey }); processPrefetchQueue(); }; // Process the prefetch queue const processPrefetchQueue = async () => { while (prefetchQueue.length > 0 && pendingPrefetchRequests.size < MAX_PREFETCH_REQUESTS) { const { query, voice, cacheKey } = prefetchQueue.shift(); const abortController = new AbortController(); pendingPrefetchRequests.set(cacheKey, abortController); const userSambanovaKey = document.getElementById('apiKey').value.trim() !== '' ? document.getElementById('apiKey').value : 'none'; const url = '/stream_text'; const requestBody = { query: query, history: JSON.stringify(conversationHistory), model: modelSelectionDropdown.value, api_key: userSambanovaKey }; try { const response = await fetch(url, { method: 'POST', headers: { 'Accept': 'text/event-stream', 'Content-Type': 'application/json' }, body: JSON.stringify(requestBody), signal: abortController.signal }); if (!response.ok) throw new Error('Network response was not ok'); const firstAudioUrl = await handleStreamingResponseForPrefetch(response.body, voice, abortController.signal); if (firstAudioUrl) prefetchCache.set(cacheKey, { url: firstAudioUrl, timestamp: Date.now() }); } catch (error) { if (error.name !== 'AbortError') console.error("Error prefetching audio:", error); } finally { pendingPrefetchRequests.delete(cacheKey); processPrefetchQueue(); } } }; // Handle the streaming response for prefetching const handleStreamingResponseForPrefetch = async (responseStream, voice, abortSignal) => { const reader = responseStream.getReader(); const decoder = new TextDecoder("utf-8"); let buffer = ""; try { while (true) { const { done, value } = await reader.read(); if (done) break; if (abortSignal.aborted) throw new DOMException('Request aborted', 'AbortError'); const chunk = decoder.decode(value, { stream: true }); buffer += chunk; const lines = buffer.split('\n'); for (let i = 0; i < lines.length - 1; i++) { const line = lines[i]; if (line.startsWith('data: ')) { const textContent = line.substring(6).trim(); if (textContent) { const audioUrl = await generateTextToSpeechAudio(textContent, voice); return audioUrl; } } } buffer = lines[lines.length - 1]; } } catch (error) { console.error("Error in handleStreamingResponseForPrefetch:", error); } finally { reader.releaseLock(); } return null; }; // Play audio from the queue const playNextAudio = async () => { if (audioPlaybackQueue.length > 0) { const audioData = audioPlaybackQueue.shift(); const audio = new Audio(audioData.url); updateActivityIndicators(); const audioPromise = new Promise(resolve => { audio.onended = resolve; audio.onerror = resolve; }); if (currentAudio) { currentAudio.pause(); currentAudio.currentTime = 0; } currentAudio = audio; await audio.play(); await audioPromise; playNextAudio(); } else { updateActivityIndicators(); } }; // Generate Text-to-Speech audio with caching const generateTextToSpeechAudio = async (text, voice) => { const normalizedText = normalizeQueryText(text); const cacheKey = `${normalizedText}-${voice}`; if (audioCache.has(cacheKey)) { const cachedData = audioCache.get(cacheKey); if (Date.now() - cachedData.timestamp < audioCacheExpiration) { return cachedData.url; } else { audioCache.delete(cacheKey); } } try { const response = await fetch(`${TEXT_TO_SPEECH_API_ENDPOINT}?voice=${voice}&text=${encodeURIComponent(text)}`, { method: 'GET' }); if (!response.ok) throw new Error('Network response was not ok'); const audioBlob = await response.blob(); const audioUrl = URL.createObjectURL(audioBlob); audioCache.set(cacheKey, { url: audioUrl, timestamp: Date.now() }); return audioUrl; } catch (error) { console.error("Error generating TTS audio:", error); return null; } }; // Send a query to the AI const sendQueryToAI = async (query) => { console.log("Sending query to AI:", query); isRequestInProgress = true; updateActivityIndicators(); firstResponseTextTimestamp = null; const normalizedQuery = normalizeQueryText(query); const cacheKey = generateCacheKey(normalizedQuery, modelSelectionDropdown.value, conversationHistory, modelSelectionDropdown.value); queryStartTime = Date.now(); if (prefetchCache.has(cacheKey)) { const cachedData = prefetchCache.get(cacheKey); if (Date.now() - cachedData.timestamp < prefetchCacheExpiration) { const prefetchedAudioUrl = cachedData.url; audioPlaybackQueue.push({ url: prefetchedAudioUrl, isPrefetched: true }); playNextAudio(); } else { prefetchCache.delete(cacheKey); } } requestAbortController = new AbortController(); const userSambanovaKey = document.getElementById('apiKey').value.trim() !== '' ? document.getElementById('apiKey').value : 'none'; const url = '/stream_text'; const requestBody = { query: query, history: JSON.stringify(conversationHistory), model: modelSelectionDropdown.value, api_key: userSambanovaKey }; try { const response = await fetch(url, { method: 'POST', headers: { 'Accept': 'text/event-stream', 'Content-Type': 'application/json' }, body: JSON.stringify(requestBody), signal: requestAbortController.signal }); if (!response.ok) { if (response.status === 429) { console.log("Rate limit hit, retrying in 1 second..."); await new Promise(resolve => setTimeout(resolve, 1000)); await sendQueryToAI(query); return; } throw new Error(`Network response was not ok: ${response.status}`); } console.log("Streaming audio response received"); await handleStreamingResponse(response.body, voiceSelectionDropdown.value, requestAbortController.signal); } catch (error) { if (error.name !== 'AbortError') { console.error("Error sending query to AI:", error); } } finally { isRequestInProgress = false; updateActivityIndicators(); } }; // Handle the streaming audio response const handleStreamingResponse = async (responseStream, voice, abortSignal) => { const reader = responseStream.getReader(); const decoder = new TextDecoder("utf-8"); let buffer = ""; let initialChunksSent = 0; let fullResponseText = ""; let fullResponseText2 = ""; let textChunk = ""; let sentText = ""; try { while (true) { const { done, value } = await reader.read(); if (done) break; if (abortSignal.aborted) throw new DOMException('Request aborted', 'AbortError'); if (isUserSpeaking) { interruptAudioPlayback('user is speaking'); break; } const chunk = decoder.decode(value, { stream: true }); buffer += chunk; const lines = buffer.split('\n'); for (let i = 0; i < lines.length - 1; i++) { const line = lines[i]; if (line.startsWith('data: ')) { const textContent = line.substring(6).trim(); if (textContent) { if (!firstResponseTextTimestamp) firstResponseTextTimestamp = Date.now(); fullResponseText += textContent + " "; fullResponseText2 += textContent + " "; textChunk += textContent + " "; transcriptDiv.textContent = fullResponseText2; if (initialChunksSent < 2) { const audioUrl = await generateTextToSpeechAudio(textContent, voice); if (audioUrl) { audioPlaybackQueue.push({ url: audioUrl, isPrefetched: false }); if (!currentAudio) playNextAudio(); } sentText += textContent + " "; initialChunksSent++; } else { let unsentTextChunk = textChunk.replace(sentText, '').trim(); if (unsentTextChunk.length >= CHUNK_SIZE) { const audioUrl = await generateTextToSpeechAudio(unsentTextChunk, voice); if (audioUrl) { audioPlaybackQueue.push({ url: audioUrl, isPrefetched: false }); if (!currentAudio) playNextAudio(); } textChunk = ""; } } if (fullResponseText !== '') { fullResponseText = ''; } } } } buffer = lines[lines.length - 1]; } } catch (error) { console.error("Error in handleStreamingResponse:", error); } finally { reader.releaseLock(); let unsentTextChunk = textChunk.replace(sentText, '').trim(); if (unsentTextChunk !== "") { const audioUrl = await generateTextToSpeechAudio(unsentTextChunk, voice); if (audioUrl) { audioPlaybackQueue.push({ url: audioUrl, isPrefetched: false }); if (!currentAudio) playNextAudio(); } } if (fullResponseText !== '') { fullResponseText = ''; } if (fullResponseText2 !== '') { addToConversationHistory('assistant', fullResponseText2); fullResponseText2 = ''; } } }; // Update activity indicators const updateActivityIndicators = (state = null) => { userActivityIndicator.textContent = isUserSpeaking ? "User: Speaking" : "User: Idle"; userActivityIndicator.className = isUserSpeaking ? "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-blue-400 to-blue-600 hover:bg-gradient-to-r from-blue-500 to-blue-700" : "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-gray-300 to-gray-400 dark:from-gray-700 dark:to-gray-800 hover:bg-gradient-to-r from-gray-400 to-gray-500"; // Tailwind classes if (isRequestInProgress && !currentAudio) { aiActivityIndicator.textContent = "AI: Processing..."; aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-purple-400 to-purple-600 hover:bg-gradient-to-r from-purple-500 to-purple-700"; // Tailwind class for thinking } else if (currentAudio && !isUserSpeaking) { aiActivityIndicator.textContent = state || "AI: Speaking"; aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-green-400 to-green-600 hover:bg-gradient-to-r from-green-500 to-green-700"; // Tailwind class for speaking } else if (isUserSpeaking) { aiActivityIndicator.textContent = "AI: Listening"; aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-yellow-400 to-yellow-600 hover:bg-gradient-to-r from-yellow-500 to-yellow-700"; // Tailwind class for listening } else { aiActivityIndicator.textContent = "AI: Idle"; aiActivityIndicator.className = "indicator rounded-full px-4 py-2 text-white flex items-center transition-colors duration-300 bg-gradient-to-r from-gray-300 to-gray-400 dark:from-gray-700 dark:to-gray-800 hover:bg-gradient-to-r from-gray-400 to-gray-500"; // Tailwind classes } }; // Initialize speech recognition if ('webkitSpeechRecognition' in window) { speechRecognizer = new webkitSpeechRecognition(); Object.assign(speechRecognizer, { continuous: true, interimResults: true, language: 'en-US', maxAlternatives: 3 }); speechRecognizer.onstart = () => { console.log("Speech recognition started"); completeTranscript = ''; isUserSpeaking = true; lastUserSpeechTimestamp = Date.now(); updateActivityIndicators(); startStopButton.innerHTML = ' Stop Listening'; }; speechRecognizer.onresult = (event) => { let interimTranscript = ''; for (let i = event.resultIndex; i < event.results.length; i++) { const transcript = event.results[i][0].transcript; if (event.results[i].isFinal) { completeTranscript += transcript; interruptAudioPlayback('final'); processSpeechTranscript(completeTranscript); completeTranscript = ''; isUserSpeaking = false; updateActivityIndicators(); queryStartTime = Date.now(); } else { interimTranscript += transcript; isUserSpeaking = true; lastUserSpeechTimestamp = Date.now(); updateActivityIndicators(); if (interimTranscript.length > prefetchTextQuery.length + 5) { cancelPrefetchRequests(prefetchTextQuery); } prefetchTextQuery = interimTranscript; prefetchFirstAudioChunk(interimTranscript, voiceSelectionDropdown.value); if (isRequestInProgress && shouldInterruptAudioPlayback(interimTranscript)) { interruptAudioPlayback('interim'); } } } }; speechRecognizer.onerror = (event) => { console.error('Speech recognition error:', event.error); if (isSpeechRecognitionActive) speechRecognizer.start(); }; speechRecognizer.onend = () => { isUserSpeaking = false; updateActivityIndicators(); if (!isRequestInProgress && completeTranscript !== '') { processSpeechTranscript(completeTranscript); completeTranscript = ''; } // Only restart if the user hasn't manually stopped the recognizer if (isSpeechRecognitionActive && !userManuallyStoppedRecognizer) { speechRecognizer.start(); } }; startStopButton.addEventListener('click', () => { if (isSpeechRecognitionActive) { speechRecognizer.stop(); isSpeechRecognitionActive = false; userManuallyStoppedRecognizer = true; // Set to true when manually stopped startStopButton.innerHTML = ' Start Listening'; } else { speechRecognizer.start(); isSpeechRecognitionActive = true; userManuallyStoppedRecognizer = false; // Set to false when started startStopButton.innerHTML = ' Stop Listening'; } }); } else { alert('Your browser does not support the Web Speech API.'); } // Add to conversation history const addToConversationHistory = (role, content) => { if (conversationHistory.length > 0 && conversationHistory[conversationHistory.length - 1].role === 'assistant' && conversationHistory[conversationHistory.length - 1].content === "") { conversationHistory.pop(); } conversationHistory.push({ role, content }); if (conversationHistory.length > 6) conversationHistory.splice(0, 2); }; // Process the final speech transcript const processSpeechTranscript = (transcript) => { const trimmedTranscript = transcript.trimStart(); if (trimmedTranscript !== '' && !isRequestInProgress) { activeQuery = trimmedTranscript; sendQueryToAI(activeQuery); addToConversationHistory('user', activeQuery); } }; // Check if audio playback should be interrupted const shouldInterruptAudioPlayback = (interimTranscript) => Date.now() - lastUserSpeechTimestamp > USER_SPEECH_INTERRUPT_DELAY || interimTranscript.length > 5; // Interrupt audio playback const interruptAudioPlayback = (reason = 'unknown') => { console.log(`Interrupting audio (reason: ${reason})...`); if (currentAudio) { currentAudio.pause(); currentAudio.currentTime = 0; currentAudio = null; } audioPlaybackQueue.length = 0; isRequestInProgress = false; if (requestAbortController) { requestAbortController.abort(); requestAbortController = null; } prefetchCache.clear(); prefetchQueue.length = 0; updateActivityIndicators(); }; // Cancel pending prefetch requests const cancelPrefetchRequests = (query) => { const normalizedQuery = normalizeQueryText(query); for (const [cacheKey, abortController] of pendingPrefetchRequests) { if (cacheKey.startsWith(normalizedQuery)) { abortController.abort(); pendingPrefetchRequests.delete(cacheKey); } } }; // Update latency display const updateLatency = () => { if (firstResponseTextTimestamp) { const latency = firstResponseTextTimestamp - queryStartTime; responseTimeDisplay.textContent = `Latency: ${latency}ms`; } else { responseTimeDisplay.textContent = "Latency: 0ms"; } }; setInterval(updateLatency, 100);