Add VoiceLoop.kt - core STT→Matrix→TTS voice pipeline wiring
This commit is contained in:
@@ -0,0 +1,253 @@
|
||||
package farm.hobbs.clippyauto
|
||||
|
||||
import android.content.Context
|
||||
import farm.hobbs.clippyauto.connectivity.MessageQueue
|
||||
import farm.hobbs.clippyauto.connectivity.NetworkMonitor
|
||||
import farm.hobbs.clippyauto.connectivity.NetworkState
|
||||
import farm.hobbs.clippyauto.matrix.MatrixRestClient
|
||||
import farm.hobbs.clippyauto.push.ClippyResponseBus
|
||||
import farm.hobbs.clippyauto.push.ResponseDedup
|
||||
import farm.hobbs.clippyauto.voice.SpeechManager
|
||||
import farm.hobbs.clippyauto.voice.TTSManager
|
||||
import farm.hobbs.clippyauto.voice.VoiceState
|
||||
import farm.hobbs.clippyauto.voice.VoiceStateMachine
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.collectLatest
|
||||
import java.security.MessageDigest
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* Core voice loop — wires STT → Matrix → sync/FCM → TTS end-to-end.
|
||||
*
|
||||
* This is the heart of ClippyAuto. It coordinates:
|
||||
* - SpeechManager (STT input)
|
||||
* - MatrixRestClient (send/receive)
|
||||
* - TTSManager (audio output)
|
||||
* - VoiceStateMachine (state transitions)
|
||||
* - NetworkMonitor (connectivity)
|
||||
* - FCM fast path (via ClippyResponseBus)
|
||||
*
|
||||
* Flow:
|
||||
* User taps talk → LISTENING → STT captures text → PROCESSING
|
||||
* → send to Matrix → wait for Clippy response (FCM or sync)
|
||||
* → RECEIVING → SPEAKING (TTS) → IDLE
|
||||
*/
|
||||
class VoiceLoop(
|
||||
private val context: Context,
|
||||
private val matrixClient: MatrixRestClient,
|
||||
private val roomId: String,
|
||||
private val clippyUserId: String = "@clippy:hobbs.farm",
|
||||
private val onStateChange: (VoiceState, String) -> Unit
|
||||
) {
|
||||
private val stateMachine = VoiceStateMachine()
|
||||
private val messageQueue = MessageQueue()
|
||||
private val networkMonitor = NetworkMonitor(context)
|
||||
private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
|
||||
|
||||
private lateinit var speechManager: SpeechManager
|
||||
private lateinit var ttsManager: TTSManager
|
||||
|
||||
private var syncJob: Job? = null
|
||||
private var sessionId: String = Instant.now().toString()
|
||||
|
||||
fun initialize() {
|
||||
// Initialize TTS
|
||||
ttsManager = TTSManager(
|
||||
context = context,
|
||||
onSpeakingStart = {
|
||||
stateMachine.transition(VoiceState.SPEAKING)
|
||||
onStateChange(VoiceState.SPEAKING, "Clippy is speaking")
|
||||
},
|
||||
onSpeakingDone = {
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, "Tap to talk to Clippy")
|
||||
}
|
||||
)
|
||||
ttsManager.initialize()
|
||||
|
||||
// Initialize STT
|
||||
speechManager = SpeechManager(
|
||||
context = context,
|
||||
onResult = { text -> onSpeechResult(text) },
|
||||
onListening = {
|
||||
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||
},
|
||||
onError = { error ->
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, error)
|
||||
}
|
||||
)
|
||||
|
||||
// Subscribe to FCM fast path
|
||||
ClippyResponseBus.subscribe { response, _ ->
|
||||
scope.launch { handleClippyResponse(response) }
|
||||
}
|
||||
|
||||
// Start network monitoring
|
||||
networkMonitor.start()
|
||||
|
||||
// Monitor network state for offline handling
|
||||
scope.launch {
|
||||
networkMonitor.state.collectLatest { state ->
|
||||
when (state) {
|
||||
NetworkState.OFFLINE -> {
|
||||
ttsManager.speak("Signal dropped. I'll catch up when you're back.")
|
||||
}
|
||||
NetworkState.ONLINE -> {
|
||||
if (!messageQueue.isEmpty) {
|
||||
ttsManager.speak("I'm back online.")
|
||||
flushQueuedMessages()
|
||||
}
|
||||
}
|
||||
NetworkState.WEAK -> { /* Continue, just slower */ }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start Matrix sync (fallback path, always running)
|
||||
startSyncLoop()
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when user taps the talk button.
|
||||
*/
|
||||
fun onTalkPressed() {
|
||||
when (stateMachine.state.value) {
|
||||
VoiceState.IDLE -> {
|
||||
stateMachine.transition(VoiceState.LISTENING)
|
||||
speechManager.startListening()
|
||||
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||
}
|
||||
VoiceState.SPEAKING -> {
|
||||
// Interruption — stop TTS, start listening
|
||||
ttsManager.interrupt()
|
||||
stateMachine.transition(VoiceState.LISTENING)
|
||||
speechManager.startListening()
|
||||
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||
}
|
||||
else -> { /* Ignore taps during PROCESSING/RECEIVING */ }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when STT produces a result.
|
||||
*/
|
||||
private fun onSpeechResult(text: String) {
|
||||
if (text.isBlank()) {
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, "Tap to talk to Clippy")
|
||||
return
|
||||
}
|
||||
|
||||
stateMachine.transition(VoiceState.PROCESSING)
|
||||
onStateChange(VoiceState.PROCESSING, "Thinking...")
|
||||
|
||||
scope.launch {
|
||||
val sent = sendToMatrix(text)
|
||||
if (!sent) {
|
||||
// Offline — queue the message
|
||||
messageQueue.enqueue(text)
|
||||
ttsManager.speak("Hang on, lost signal for a sec.")
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, "Offline — message queued")
|
||||
}
|
||||
// If sent, we wait for response via FCM or sync
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send transcribed text to Matrix #clippy-voice room.
|
||||
*/
|
||||
private suspend fun sendToMatrix(text: String): Boolean {
|
||||
return withContext(Dispatchers.IO) {
|
||||
matrixClient.sendMessage(
|
||||
roomId = roomId,
|
||||
body = text,
|
||||
source = "voice",
|
||||
sessionId = sessionId
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Clippy's response — from either FCM or Matrix sync.
|
||||
*/
|
||||
private fun handleClippyResponse(text: String) {
|
||||
// Dedup — mark this response so the other path skips it
|
||||
val hash = sha256(text)
|
||||
if (ResponseDedup.seen(hash)) return
|
||||
ResponseDedup.mark(hash)
|
||||
|
||||
stateMachine.transition(VoiceState.RECEIVING)
|
||||
onStateChange(VoiceState.RECEIVING, "")
|
||||
|
||||
// Send to TTS — TTSManager handles queuing if multiple chunks arrive
|
||||
ttsManager.speak(text)
|
||||
}
|
||||
|
||||
/**
|
||||
* Background sync loop — polls Matrix for messages.
|
||||
* This is the fallback path when FCM doesn't deliver first.
|
||||
*/
|
||||
private fun startSyncLoop() {
|
||||
syncJob = scope.launch(Dispatchers.IO) {
|
||||
while (isActive) {
|
||||
try {
|
||||
val messages = matrixClient.sync(roomId, timeoutMs = 30000)
|
||||
messages
|
||||
.filter { it.sender == clippyUserId }
|
||||
.filter { it.signal == null } // Skip session signals
|
||||
.forEach { msg ->
|
||||
withContext(Dispatchers.Main) {
|
||||
handleClippyResponse(msg.body)
|
||||
}
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
delay(5000) // Back off on error
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Flush messages queued during offline period.
|
||||
*/
|
||||
private suspend fun flushQueuedMessages() {
|
||||
val queued = messageQueue.flush()
|
||||
for (msg in queued) {
|
||||
sendToMatrix(msg.text)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send session start signal.
|
||||
*/
|
||||
fun startSession() {
|
||||
sessionId = Instant.now().toString()
|
||||
scope.launch(Dispatchers.IO) {
|
||||
matrixClient.sendSessionSignal(roomId, "session_start", sessionId)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send session end signal.
|
||||
*/
|
||||
fun endSession() {
|
||||
scope.launch(Dispatchers.IO) {
|
||||
matrixClient.sendSessionSignal(roomId, "session_end", sessionId)
|
||||
}
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
syncJob?.cancel()
|
||||
speechManager.destroy()
|
||||
ttsManager.shutdown()
|
||||
networkMonitor.stop()
|
||||
scope.cancel()
|
||||
}
|
||||
|
||||
private fun sha256(text: String): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256")
|
||||
return digest.digest(text.toByteArray()).joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user