From a6f8433eb441b874590167e32d97585aed732218 Mon Sep 17 00:00:00 2001 From: Spirit Date: Thu, 5 Mar 2026 23:09:19 +0000 Subject: [PATCH] =?UTF-8?q?Add=20VoiceLoop.kt=20-=20core=20STT=E2=86=92Mat?= =?UTF-8?q?rix=E2=86=92TTS=20voice=20pipeline=20wiring?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/farm/hobbs/clippyauto/VoiceLoop.kt | 253 ++++++++++++++++++ 1 file changed, 253 insertions(+) create mode 100644 app/src/main/java/farm/hobbs/clippyauto/VoiceLoop.kt diff --git a/app/src/main/java/farm/hobbs/clippyauto/VoiceLoop.kt b/app/src/main/java/farm/hobbs/clippyauto/VoiceLoop.kt new file mode 100644 index 0000000..b5ff219 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/VoiceLoop.kt @@ -0,0 +1,253 @@ +package farm.hobbs.clippyauto + +import android.content.Context +import farm.hobbs.clippyauto.connectivity.MessageQueue +import farm.hobbs.clippyauto.connectivity.NetworkMonitor +import farm.hobbs.clippyauto.connectivity.NetworkState +import farm.hobbs.clippyauto.matrix.MatrixRestClient +import farm.hobbs.clippyauto.push.ClippyResponseBus +import farm.hobbs.clippyauto.push.ResponseDedup +import farm.hobbs.clippyauto.voice.SpeechManager +import farm.hobbs.clippyauto.voice.TTSManager +import farm.hobbs.clippyauto.voice.VoiceState +import farm.hobbs.clippyauto.voice.VoiceStateMachine +import kotlinx.coroutines.* +import kotlinx.coroutines.flow.collectLatest +import java.security.MessageDigest +import java.time.Instant + +/** + * Core voice loop — wires STT → Matrix → sync/FCM → TTS end-to-end. + * + * This is the heart of ClippyAuto. It coordinates: + * - SpeechManager (STT input) + * - MatrixRestClient (send/receive) + * - TTSManager (audio output) + * - VoiceStateMachine (state transitions) + * - NetworkMonitor (connectivity) + * - FCM fast path (via ClippyResponseBus) + * + * Flow: + * User taps talk → LISTENING → STT captures text → PROCESSING + * → send to Matrix → wait for Clippy response (FCM or sync) + * → RECEIVING → SPEAKING (TTS) → IDLE + */ +class VoiceLoop( + private val context: Context, + private val matrixClient: MatrixRestClient, + private val roomId: String, + private val clippyUserId: String = "@clippy:hobbs.farm", + private val onStateChange: (VoiceState, String) -> Unit +) { + private val stateMachine = VoiceStateMachine() + private val messageQueue = MessageQueue() + private val networkMonitor = NetworkMonitor(context) + private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob()) + + private lateinit var speechManager: SpeechManager + private lateinit var ttsManager: TTSManager + + private var syncJob: Job? = null + private var sessionId: String = Instant.now().toString() + + fun initialize() { + // Initialize TTS + ttsManager = TTSManager( + context = context, + onSpeakingStart = { + stateMachine.transition(VoiceState.SPEAKING) + onStateChange(VoiceState.SPEAKING, "Clippy is speaking") + }, + onSpeakingDone = { + stateMachine.transition(VoiceState.IDLE) + onStateChange(VoiceState.IDLE, "Tap to talk to Clippy") + } + ) + ttsManager.initialize() + + // Initialize STT + speechManager = SpeechManager( + context = context, + onResult = { text -> onSpeechResult(text) }, + onListening = { + onStateChange(VoiceState.LISTENING, "Listening...") + }, + onError = { error -> + stateMachine.transition(VoiceState.IDLE) + onStateChange(VoiceState.IDLE, error) + } + ) + + // Subscribe to FCM fast path + ClippyResponseBus.subscribe { response, _ -> + scope.launch { handleClippyResponse(response) } + } + + // Start network monitoring + networkMonitor.start() + + // Monitor network state for offline handling + scope.launch { + networkMonitor.state.collectLatest { state -> + when (state) { + NetworkState.OFFLINE -> { + ttsManager.speak("Signal dropped. I'll catch up when you're back.") + } + NetworkState.ONLINE -> { + if (!messageQueue.isEmpty) { + ttsManager.speak("I'm back online.") + flushQueuedMessages() + } + } + NetworkState.WEAK -> { /* Continue, just slower */ } + } + } + } + + // Start Matrix sync (fallback path, always running) + startSyncLoop() + } + + /** + * Called when user taps the talk button. + */ + fun onTalkPressed() { + when (stateMachine.state.value) { + VoiceState.IDLE -> { + stateMachine.transition(VoiceState.LISTENING) + speechManager.startListening() + onStateChange(VoiceState.LISTENING, "Listening...") + } + VoiceState.SPEAKING -> { + // Interruption — stop TTS, start listening + ttsManager.interrupt() + stateMachine.transition(VoiceState.LISTENING) + speechManager.startListening() + onStateChange(VoiceState.LISTENING, "Listening...") + } + else -> { /* Ignore taps during PROCESSING/RECEIVING */ } + } + } + + /** + * Called when STT produces a result. + */ + private fun onSpeechResult(text: String) { + if (text.isBlank()) { + stateMachine.transition(VoiceState.IDLE) + onStateChange(VoiceState.IDLE, "Tap to talk to Clippy") + return + } + + stateMachine.transition(VoiceState.PROCESSING) + onStateChange(VoiceState.PROCESSING, "Thinking...") + + scope.launch { + val sent = sendToMatrix(text) + if (!sent) { + // Offline — queue the message + messageQueue.enqueue(text) + ttsManager.speak("Hang on, lost signal for a sec.") + stateMachine.transition(VoiceState.IDLE) + onStateChange(VoiceState.IDLE, "Offline — message queued") + } + // If sent, we wait for response via FCM or sync + } + } + + /** + * Send transcribed text to Matrix #clippy-voice room. + */ + private suspend fun sendToMatrix(text: String): Boolean { + return withContext(Dispatchers.IO) { + matrixClient.sendMessage( + roomId = roomId, + body = text, + source = "voice", + sessionId = sessionId + ) + } + } + + /** + * Handle Clippy's response — from either FCM or Matrix sync. + */ + private fun handleClippyResponse(text: String) { + // Dedup — mark this response so the other path skips it + val hash = sha256(text) + if (ResponseDedup.seen(hash)) return + ResponseDedup.mark(hash) + + stateMachine.transition(VoiceState.RECEIVING) + onStateChange(VoiceState.RECEIVING, "") + + // Send to TTS — TTSManager handles queuing if multiple chunks arrive + ttsManager.speak(text) + } + + /** + * Background sync loop — polls Matrix for messages. + * This is the fallback path when FCM doesn't deliver first. + */ + private fun startSyncLoop() { + syncJob = scope.launch(Dispatchers.IO) { + while (isActive) { + try { + val messages = matrixClient.sync(roomId, timeoutMs = 30000) + messages + .filter { it.sender == clippyUserId } + .filter { it.signal == null } // Skip session signals + .forEach { msg -> + withContext(Dispatchers.Main) { + handleClippyResponse(msg.body) + } + } + } catch (e: Exception) { + delay(5000) // Back off on error + } + } + } + } + + /** + * Flush messages queued during offline period. + */ + private suspend fun flushQueuedMessages() { + val queued = messageQueue.flush() + for (msg in queued) { + sendToMatrix(msg.text) + } + } + + /** + * Send session start signal. + */ + fun startSession() { + sessionId = Instant.now().toString() + scope.launch(Dispatchers.IO) { + matrixClient.sendSessionSignal(roomId, "session_start", sessionId) + } + } + + /** + * Send session end signal. + */ + fun endSession() { + scope.launch(Dispatchers.IO) { + matrixClient.sendSessionSignal(roomId, "session_end", sessionId) + } + } + + fun destroy() { + syncJob?.cancel() + speechManager.destroy() + ttsManager.shutdown() + networkMonitor.stop() + scope.cancel() + } + + private fun sha256(text: String): String { + val digest = MessageDigest.getInstance("SHA-256") + return digest.digest(text.toByteArray()).joinToString("") { "%02x".format(it) } + } +}