/** * @fileoverview Voice input with Deepgram Nova-3 (primary) and Web Speech API (fallback). * * Defines two singleton objects: * * - DeepgramProvider — Direct browser-to-Deepgram WebSocket connection for speech-to-text. * Captures audio via MediaRecorder, streams chunks every 250ms, handles KeepAlive pings, * auto-detects MIME type (opus/webm/mp4), and supports custom key terms for dev vocabulary. * * - VoiceInput — High-level voice input controller. Toggle mode: tap mic to start, tap * again to stop. Auto-stops after 3s silence. Shows floating preview overlay with recording * indicator, level meter (AnalyserNode), and elapsed timer. Two insert modes: "direct" * (inject into local echo overlay or PTY) and "compose" (editable textarea overlay). * Includes a temporary green Send button that replaces the settings gear icon after voice input. * Web Speech API has auto-retry (up to 2x) for premature onend and iOS Safari stability check. * * @globals {object} DeepgramProvider * @globals {object} VoiceInput * * @dependency mobile-handlers.js (MobileDetection for device checks) * @dependency app.js (uses global `app` for sendInput, showToast, terminal focus) * @loadorder 3 of 15 — loaded after mobile-handlers.js, before notification-manager.js */ // Codeman — Voice input with Deepgram Nova-3 and Web Speech API fallback // Loaded after mobile-handlers.js, before app.js // ═══════════════════════════════════════════════════════════════ // Voice Input (Deepgram Nova-3 + Web Speech API fallback) // ═══════════════════════════════════════════════════════════════ /** * DeepgramProvider - Speech-to-text via Deepgram Nova-3 WebSocket API. * Direct browser-to-Deepgram connection (no server proxy). * Uses MediaRecorder to capture audio and streams via WebSocket. */ const DeepgramProvider = { _ws: null, _mediaRecorder: null, _stream: null, _silenceTimeout: null, _keepAliveInterval: null, _onResult: null, _onError: null, _onEnd: null, /** * Start streaming audio to Deepgram. * @param {object} opts - { apiKey, language, keyterms[], onResult(text, isFinal), onError(msg), onEnd(), onStream(stream) } */ async start(opts) { this._onResult = opts.onResult; this._onError = opts.onError; this._onEnd = opts.onEnd; // 1. Get microphone access if (!navigator.mediaDevices?.getUserMedia) { this._onError?.('Microphone requires a secure context (HTTPS). Use --https flag or access via localhost.'); this._cleanup(); return; } try { this._stream = await navigator.mediaDevices.getUserMedia({ audio: { noiseSuppression: true, echoCancellation: true, autoGainControl: true } }); } catch (err) { const msg = err.name === 'NotAllowedError' ? 'Microphone access denied. Check browser settings.' : 'Microphone error: ' + err.message; this._onError?.(msg); this._cleanup(); return; } // Notify caller so it can set up audio level meter opts.onStream?.(this._stream); // 2. Detect best supported MIME type for MediaRecorder const mimeTypes = ['audio/webm;codecs=opus', 'audio/webm', 'audio/mp4']; this._selectedMime = null; for (const mt of mimeTypes) { if (typeof MediaRecorder !== 'undefined' && MediaRecorder.isTypeSupported(mt)) { this._selectedMime = mt; break; } } // 3. Build WebSocket URL (no encoding param — Deepgram auto-detects from container format) const params = new URLSearchParams({ model: 'nova-3', smart_format: 'false', punctuate: 'false', interim_results: 'true', utterance_end_ms: '1500', vad_events: 'true', }); if (opts.language && opts.language !== 'multi') { params.set('language', opts.language); } else if (opts.language === 'multi') { params.set('detect_language', 'true'); } if (opts.keyterms?.length) { for (const term of opts.keyterms) { const trimmed = term.trim(); if (trimmed) params.append('keyterm', trimmed + ':2'); } } // 4. Connect WebSocket (trim API key to avoid whitespace auth failures) const apiKey = (opts.apiKey || '').trim(); if (!apiKey) { this._onError?.('No Deepgram API key configured. Add one in Settings > Voice.'); this._cleanup(); return; } const wsUrl = `wss://api.deepgram.com/v1/listen?${params}`; try { this._ws = new WebSocket(wsUrl, ['token', apiKey]); } catch (err) { this._onError?.('Failed to connect to Deepgram: ' + err.message); this._cleanup(); return; } this._ws.onopen = () => { // 5. Send KeepAlive every 8s to prevent Deepgram from closing idle connections // (covers the gap before MediaRecorder produces its first chunk) this._keepAliveInterval = setInterval(() => { if (this._ws?.readyState === WebSocket.OPEN) { try { this._ws.send(JSON.stringify({ type: 'KeepAlive' })); } catch (_e) { /* ignore */ } } }, 8000); // 6. Start MediaRecorder once connected this._startRecording(); }; this._ws.onmessage = (event) => { try { const data = JSON.parse(event.data); if (data.type === 'Results' && data.channel?.alternatives?.[0]) { const alt = data.channel.alternatives[0]; const transcript = alt.transcript || ''; if (transcript) { const isFinal = data.is_final === true; this._onResult?.(transcript, isFinal); this._resetSilenceTimeout(); } } } catch (_e) { // Ignore parse errors for non-JSON messages } }; this._ws.onerror = () => { // WebSocket onerror doesn't carry useful info — onclose handles it }; this._ws.onclose = (event) => { clearInterval(this._keepAliveInterval); this._keepAliveInterval = null; if (event.code === 1008) { this._onError?.('Authentication failed. Check your Deepgram API key in Settings > Voice.'); } else if (event.code === 1006) { // 1006 = abnormal closure (no close frame). Usually auth failure, expired key, or no credits. this._onError?.('Deepgram connection failed (1006). Check your API key is valid and has credits in Settings > Voice.'); } else if (event.code !== 1000) { this._onError?.('Deepgram connection closed: ' + (event.reason || `code ${event.code}`)); } this._stopRecording(); this._onEnd?.(); }; }, _startRecording() { if (!this._stream || !this._ws || this._ws.readyState !== WebSocket.OPEN) return; const recorderOpts = this._selectedMime ? { mimeType: this._selectedMime } : {}; try { this._mediaRecorder = new MediaRecorder(this._stream, recorderOpts); } catch (err) { this._onError?.('MediaRecorder failed: ' + err.message); this._cleanup(); return; } this._mediaRecorder.ondataavailable = (event) => { if (event.data.size > 0 && this._ws?.readyState === WebSocket.OPEN) { this._ws.send(event.data); } }; this._mediaRecorder.start(250); // Send chunks every 250ms this._resetSilenceTimeout(); }, _stopRecording() { if (this._mediaRecorder && this._mediaRecorder.state !== 'inactive') { try { this._mediaRecorder.stop(); } catch (_e) { /* already stopped */ } } // Stop all mic tracks if (this._stream) { this._stream.getTracks().forEach(t => t.stop()); } }, _resetSilenceTimeout() { clearTimeout(this._silenceTimeout); this._silenceTimeout = setTimeout(() => { this.stop(); }, 3000); }, stop() { clearTimeout(this._silenceTimeout); this._silenceTimeout = null; clearInterval(this._keepAliveInterval); this._keepAliveInterval = null; this._stopRecording(); // Detach WS handlers before closing to prevent stale onclose from // killing a subsequent recording that starts before the close completes if (this._ws) { this._ws.onclose = null; this._ws.onmessage = null; this._ws.onerror = null; if (this._ws.readyState === WebSocket.OPEN) { try { this._ws.close(1000); } catch (_e) { /* ignore */ } } this._ws = null; } // Save onEnd before nulling — must notify VoiceInput when silence timeout // triggers stop internally (VoiceInput.onEnd guards with isRecording check) const onEnd = this._onEnd; this._onResult = null; this._onError = null; this._onEnd = null; onEnd?.(); }, _cleanup() { this.stop(); this._mediaRecorder = null; this._stream = null; this._selectedMime = null; } }; /** * VoiceInput - Speech-to-text with Deepgram Nova-3 (primary) and Web Speech API (fallback). * Toggle mode: tap mic to start, tap again to stop. Auto-stops after silence. * Shows interim transcription in a floating preview overlay. * Inserts final text into the active session (user presses Enter to submit). */ const VoiceInput = { recognition: null, isRecording: false, supported: false, silenceTimeout: null, previewEl: null, _lastTranscript: '', _stabilityTimer: null, _accumulatedFinal: '', _activeProvider: null, // 'deepgram' | 'webspeech' | null _recordingStartedAt: 0, // timestamp when recording started _retryCount: 0, // auto-retry counter for premature Web Speech API ends _hasReceivedResult: false, // whether any speech result came in this session _durationInterval: null, // timer for updating elapsed time display _analyser: null, // AudioContext analyser for level meter _analyserSource: null, // MediaStreamSource for level meter _audioContext: null, // AudioContext for level meter _levelAnimFrame: null, // rAF handle for level meter init() { this._initRecognition(); // Always show buttons — if unsupported, toggle() shows a toast this._showButtons(); }, // --- Deepgram config (localStorage only, never sent to server) --- _getDeepgramConfig() { try { return JSON.parse(localStorage.getItem('codeman-voice-settings') || '{}'); } catch (_e) { return {}; } }, _saveDeepgramConfig(config) { localStorage.setItem('codeman-voice-settings', JSON.stringify(config)); }, _shouldUseDeepgram() { const cfg = this._getDeepgramConfig(); return !!(cfg.apiKey && cfg.apiKey.trim()); }, /** Get the active provider name for display */ getActiveProviderName() { if (this._shouldUseDeepgram()) return 'Deepgram Nova-3'; if (this.supported) return 'Web Speech API'; return 'None'; }, /** Try to create a SpeechRecognition instance */ _initRecognition() { const SR = window.SpeechRecognition || window.webkitSpeechRecognition; this.supported = !!SR; if (!this.supported) return; this.recognition = new SR(); this.recognition.continuous = true; this.recognition.interimResults = true; this.recognition.lang = 'en-US'; this.recognition.maxAlternatives = 1; this.recognition.onresult = (e) => this._onWebSpeechResult(e); this.recognition.onerror = (e) => this._onWebSpeechError(e); this.recognition.onend = () => this._onWebSpeechEnd(); }, toggle() { if (this.isRecording) { this.stop(); } else { this.start(); } }, start() { if (this.isRecording) return; if (!app.activeSessionId) { app.showToast('No active session', 'warning'); return; } this._retryCount = 0; if (this._shouldUseDeepgram()) { this._startDeepgram(); } else { this._startWebSpeech(); } }, _startDeepgram() { const cfg = this._getDeepgramConfig(); this.isRecording = true; this._activeProvider = 'deepgram'; this._accumulatedFinal = ''; this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); this._updateButtons('recording'); this._showPreview('Listening...', 'deepgram'); this._startDurationTimer(); const keyterms = (cfg.keyterms || 'refactor, endpoint, middleware, callback, async, regex, TypeScript, npm, API, deploy, config, linter, env, webhook, schema, CLI, JSON, CSS, DOM, SSE, backend, frontend, localhost, dependencies, repository, merge, rebase, diff, commit, com') .split(',').map(t => t.trim()).filter(Boolean); DeepgramProvider.start({ apiKey: cfg.apiKey, language: cfg.language || 'en-US', keyterms, onStream: (stream) => { this._startLevelMeter(stream); }, onResult: (text, isFinal) => { if (!this.isRecording) return; this._hasReceivedResult = true; if (isFinal) { this._accumulatedFinal += text; this._hidePreview(); this._insertText(this._accumulatedFinal); this.stop(); } else { const display = this._accumulatedFinal + text; this._showPreview(display, 'deepgram'); } }, onError: (msg) => { const wasRecording = this.isRecording; this.stop(); if (wasRecording) app.showToast(msg, 'error'); }, onEnd: () => { if (this.isRecording) { if (this._accumulatedFinal) { this._insertText(this._accumulatedFinal); } this.stop(); } } }); // Haptic feedback on mobile if (navigator.vibrate) navigator.vibrate(50); }, _startWebSpeech() { // Lazy-init: retry if recognition was cleaned up or not available at page load if (!this.recognition) this._initRecognition(); if (!this.supported) { if (!this._shouldUseDeepgram()) { app.showToast('Voice input not available. Configure Deepgram in Settings > Voice.', 'warning'); } else { app.showToast('Voice input not supported in this browser', 'warning'); } return; } this.isRecording = true; this._activeProvider = 'webspeech'; this._accumulatedFinal = ''; this._lastTranscript = ''; this._hasReceivedResult = false; this._recordingStartedAt = Date.now(); this._updateButtons('recording'); this._showPreview('Listening...'); this._startDurationTimer(); try { this.recognition.start(); } catch (e) { // InvalidStateError = already started — ignore. Other errors = genuine failure. if (e.name !== 'InvalidStateError') { this.stop(); app.showToast('Voice input failed to start: ' + e.message, 'error'); return; } } this._resetSilenceTimeout(); // Get mic stream for level meter (non-blocking — level meter is cosmetic) navigator.mediaDevices?.getUserMedia({ audio: true }).then(stream => { if (this.isRecording && this._activeProvider === 'webspeech') { this._webSpeechStream = stream; this._startLevelMeter(stream); } else { stream.getTracks().forEach(t => t.stop()); } }).catch(() => { /* level meter just won't show */ }); // Haptic feedback on mobile if (navigator.vibrate) navigator.vibrate(50); }, stop() { if (!this.isRecording) return; this.isRecording = false; clearTimeout(this.silenceTimeout); clearTimeout(this._stabilityTimer); this.silenceTimeout = null; this._stabilityTimer = null; this._retryCount = 0; this._stopDurationTimer(); this._stopLevelMeter(); this._updateButtons('idle'); this._hidePreview(); if (this._activeProvider === 'deepgram') { DeepgramProvider.stop(); } else if (this._activeProvider === 'webspeech') { try { this.recognition?.stop(); } catch (_e) { // Already stopped — ignore } // Stop the mic stream we opened for the level meter if (this._webSpeechStream) { this._webSpeechStream.getTracks().forEach(t => t.stop()); this._webSpeechStream = null; } } this._activeProvider = null; // Haptic feedback on mobile if (navigator.vibrate) navigator.vibrate([30, 50, 30]); }, _onWebSpeechResult(event) { if (!this.isRecording) return; this._hasReceivedResult = true; this._resetSilenceTimeout(); let interim = ''; let finalText = ''; for (let i = event.resultIndex; i < event.results.length; i++) { const transcript = event.results[i][0].transcript; if (event.results[i].isFinal) { finalText += transcript; } else { interim += transcript; } } if (finalText) { this._accumulatedFinal += finalText; this._hidePreview(); this._insertText(this._accumulatedFinal); this.stop(); } else if (interim) { const display = this._accumulatedFinal + interim; this._showPreview(display); // iOS Safari workaround: isFinal is always false. // Detect when interim results stop changing for 750ms → treat as final. this._iosStabilityCheck(interim); } }, _onWebSpeechError(event) { // During auto-retry, 'aborted' and 'no-speech' errors are expected — ignore them if (this._retryCount > 0 && (event.error === 'aborted' || event.error === 'no-speech')) return; const wasRecording = this.isRecording; this.stop(); if (!wasRecording) return; switch (event.error) { case 'not-allowed': app.showToast('Microphone access denied. Check browser settings.', 'error'); break; case 'no-speech': // Silent — auto-stop is enough feedback break; case 'network': app.showToast('Voice input requires internet connection.', 'error'); break; case 'aborted': // User cancelled — no message needed break; default: app.showToast('Voice input error: ' + event.error, 'error'); } }, _onWebSpeechEnd() { // Recognition ended (browser auto-stopped or we called stop()) if (!this.isRecording) return; const elapsed = Date.now() - this._recordingStartedAt; // Web Speech API often fires onend prematurely on the first attempt (< 500ms, no results). // Auto-retry up to 2 times to avoid the "needs two clicks" problem. if (elapsed < 500 && !this._hasReceivedResult && this._retryCount < 2) { this._retryCount++; try { this.recognition.start(); } catch (_e) { // If restart fails, fall through to stop if (this._accumulatedFinal) this._insertText(this._accumulatedFinal); this.stop(); } return; } // Genuine end — finalize any accumulated text if (this._accumulatedFinal) { this._insertText(this._accumulatedFinal); } this.stop(); }, _insertText(text) { if (!app.activeSessionId || !text.trim()) return; const trimmed = text.trim(); const mode = this._getDeepgramConfig().insertMode || 'direct'; if (mode === 'compose') { // If a compose overlay is already open, populate its textarea instead of recreating const existingTextarea = document.querySelector('.voice-compose-overlay .paste-textarea'); if (existingTextarea) { existingTextarea.value = trimmed; existingTextarea.focus(); existingTextarea.selectionStart = existingTextarea.selectionEnd = trimmed.length; } else { this._showComposeOverlay(trimmed); } } else { // Direct mode: inject into local echo overlay if available, else send to PTY if (app._localEchoEnabled && app._localEchoOverlay) { app._localEchoOverlay.appendText(trimmed); } else { app.sendInput(trimmed).catch(() => {}); } this._showVoiceSendBtn(); setTimeout(() => { if (app.terminal) app.terminal.focus(); }, 150); } }, /** Show a green Enter button by transforming the gear icon in-place */ _showVoiceSendBtn() { // Find the gear button (mobile or desktop header) const gear = document.querySelector('.btn-settings-mobile') || document.querySelector('.btn-settings'); if (!gear || gear.classList.contains('voice-send-active')) return; // Remove existing if any this._hideVoiceSendBtn(); // Save original state this._voiceSendGear = gear; this._voiceSendOriginalHTML = gear.innerHTML; this._voiceSendOriginalOnclick = gear.getAttribute('onclick'); // Transform into green send button gear.innerHTML = ''; gear.classList.add('voice-send-active'); gear.removeAttribute('onclick'); gear.title = 'Send (Enter)'; // Click handler this._voiceSendHandler = () => { if (!app.activeSessionId) return; // Simulate Enter key: if local echo is active, flush its buffer + send \r; // otherwise just send \r directly to the PTY if (app._localEchoEnabled && app._localEchoOverlay) { const text = app._localEchoOverlay.pendingText || ''; app._localEchoOverlay.clear(); app._localEchoOverlay.suppressBufferDetection(); if (text) app.sendInput(text).catch(() => {}); setTimeout(() => app.sendInput('\r').catch(() => {}), 80); } else { app.sendInput('\r').catch(() => {}); } // Blink then restore gear.classList.add('voice-send-blink'); setTimeout(() => this._hideVoiceSendBtn(), 400); }; gear.addEventListener('click', this._voiceSendHandler); }, _hideVoiceSendBtn() { const gear = this._voiceSendGear; if (!gear) return; gear.removeEventListener('click', this._voiceSendHandler); gear.classList.remove('voice-send-active', 'voice-send-blink'); gear.innerHTML = this._voiceSendOriginalHTML || ''; if (this._voiceSendOriginalOnclick) { gear.setAttribute('onclick', this._voiceSendOriginalOnclick); } gear.title = 'App Settings'; this._voiceSendGear = null; this._voiceSendHandler = null; this._voiceSendOriginalHTML = null; this._voiceSendOriginalOnclick = null; }, /** Show an editable compose overlay so the user can review/edit before sending */ _showComposeOverlay(text) { document.querySelector('.voice-compose-overlay')?.remove(); const overlay = document.createElement('div'); overlay.className = 'voice-compose-overlay paste-overlay'; overlay.innerHTML = `