Add VoiceLoop.kt - core STT→Matrix→TTS voice pipeline wiring
This commit is contained in:
@@ -0,0 +1,253 @@
|
|||||||
|
package farm.hobbs.clippyauto
|
||||||
|
|
||||||
|
import android.content.Context
|
||||||
|
import farm.hobbs.clippyauto.connectivity.MessageQueue
|
||||||
|
import farm.hobbs.clippyauto.connectivity.NetworkMonitor
|
||||||
|
import farm.hobbs.clippyauto.connectivity.NetworkState
|
||||||
|
import farm.hobbs.clippyauto.matrix.MatrixRestClient
|
||||||
|
import farm.hobbs.clippyauto.push.ClippyResponseBus
|
||||||
|
import farm.hobbs.clippyauto.push.ResponseDedup
|
||||||
|
import farm.hobbs.clippyauto.voice.SpeechManager
|
||||||
|
import farm.hobbs.clippyauto.voice.TTSManager
|
||||||
|
import farm.hobbs.clippyauto.voice.VoiceState
|
||||||
|
import farm.hobbs.clippyauto.voice.VoiceStateMachine
|
||||||
|
import kotlinx.coroutines.*
|
||||||
|
import kotlinx.coroutines.flow.collectLatest
|
||||||
|
import java.security.MessageDigest
|
||||||
|
import java.time.Instant
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Core voice loop — wires STT → Matrix → sync/FCM → TTS end-to-end.
|
||||||
|
*
|
||||||
|
* This is the heart of ClippyAuto. It coordinates:
|
||||||
|
* - SpeechManager (STT input)
|
||||||
|
* - MatrixRestClient (send/receive)
|
||||||
|
* - TTSManager (audio output)
|
||||||
|
* - VoiceStateMachine (state transitions)
|
||||||
|
* - NetworkMonitor (connectivity)
|
||||||
|
* - FCM fast path (via ClippyResponseBus)
|
||||||
|
*
|
||||||
|
* Flow:
|
||||||
|
* User taps talk → LISTENING → STT captures text → PROCESSING
|
||||||
|
* → send to Matrix → wait for Clippy response (FCM or sync)
|
||||||
|
* → RECEIVING → SPEAKING (TTS) → IDLE
|
||||||
|
*/
|
||||||
|
class VoiceLoop(
|
||||||
|
private val context: Context,
|
||||||
|
private val matrixClient: MatrixRestClient,
|
||||||
|
private val roomId: String,
|
||||||
|
private val clippyUserId: String = "@clippy:hobbs.farm",
|
||||||
|
private val onStateChange: (VoiceState, String) -> Unit
|
||||||
|
) {
|
||||||
|
private val stateMachine = VoiceStateMachine()
|
||||||
|
private val messageQueue = MessageQueue()
|
||||||
|
private val networkMonitor = NetworkMonitor(context)
|
||||||
|
private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
|
||||||
|
|
||||||
|
private lateinit var speechManager: SpeechManager
|
||||||
|
private lateinit var ttsManager: TTSManager
|
||||||
|
|
||||||
|
private var syncJob: Job? = null
|
||||||
|
private var sessionId: String = Instant.now().toString()
|
||||||
|
|
||||||
|
fun initialize() {
|
||||||
|
// Initialize TTS
|
||||||
|
ttsManager = TTSManager(
|
||||||
|
context = context,
|
||||||
|
onSpeakingStart = {
|
||||||
|
stateMachine.transition(VoiceState.SPEAKING)
|
||||||
|
onStateChange(VoiceState.SPEAKING, "Clippy is speaking")
|
||||||
|
},
|
||||||
|
onSpeakingDone = {
|
||||||
|
stateMachine.transition(VoiceState.IDLE)
|
||||||
|
onStateChange(VoiceState.IDLE, "Tap to talk to Clippy")
|
||||||
|
}
|
||||||
|
)
|
||||||
|
ttsManager.initialize()
|
||||||
|
|
||||||
|
// Initialize STT
|
||||||
|
speechManager = SpeechManager(
|
||||||
|
context = context,
|
||||||
|
onResult = { text -> onSpeechResult(text) },
|
||||||
|
onListening = {
|
||||||
|
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||||
|
},
|
||||||
|
onError = { error ->
|
||||||
|
stateMachine.transition(VoiceState.IDLE)
|
||||||
|
onStateChange(VoiceState.IDLE, error)
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
// Subscribe to FCM fast path
|
||||||
|
ClippyResponseBus.subscribe { response, _ ->
|
||||||
|
scope.launch { handleClippyResponse(response) }
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start network monitoring
|
||||||
|
networkMonitor.start()
|
||||||
|
|
||||||
|
// Monitor network state for offline handling
|
||||||
|
scope.launch {
|
||||||
|
networkMonitor.state.collectLatest { state ->
|
||||||
|
when (state) {
|
||||||
|
NetworkState.OFFLINE -> {
|
||||||
|
ttsManager.speak("Signal dropped. I'll catch up when you're back.")
|
||||||
|
}
|
||||||
|
NetworkState.ONLINE -> {
|
||||||
|
if (!messageQueue.isEmpty) {
|
||||||
|
ttsManager.speak("I'm back online.")
|
||||||
|
flushQueuedMessages()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
NetworkState.WEAK -> { /* Continue, just slower */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start Matrix sync (fallback path, always running)
|
||||||
|
startSyncLoop()
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when user taps the talk button.
|
||||||
|
*/
|
||||||
|
fun onTalkPressed() {
|
||||||
|
when (stateMachine.state.value) {
|
||||||
|
VoiceState.IDLE -> {
|
||||||
|
stateMachine.transition(VoiceState.LISTENING)
|
||||||
|
speechManager.startListening()
|
||||||
|
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||||
|
}
|
||||||
|
VoiceState.SPEAKING -> {
|
||||||
|
// Interruption — stop TTS, start listening
|
||||||
|
ttsManager.interrupt()
|
||||||
|
stateMachine.transition(VoiceState.LISTENING)
|
||||||
|
speechManager.startListening()
|
||||||
|
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||||
|
}
|
||||||
|
else -> { /* Ignore taps during PROCESSING/RECEIVING */ }
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Called when STT produces a result.
|
||||||
|
*/
|
||||||
|
private fun onSpeechResult(text: String) {
|
||||||
|
if (text.isBlank()) {
|
||||||
|
stateMachine.transition(VoiceState.IDLE)
|
||||||
|
onStateChange(VoiceState.IDLE, "Tap to talk to Clippy")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
stateMachine.transition(VoiceState.PROCESSING)
|
||||||
|
onStateChange(VoiceState.PROCESSING, "Thinking...")
|
||||||
|
|
||||||
|
scope.launch {
|
||||||
|
val sent = sendToMatrix(text)
|
||||||
|
if (!sent) {
|
||||||
|
// Offline — queue the message
|
||||||
|
messageQueue.enqueue(text)
|
||||||
|
ttsManager.speak("Hang on, lost signal for a sec.")
|
||||||
|
stateMachine.transition(VoiceState.IDLE)
|
||||||
|
onStateChange(VoiceState.IDLE, "Offline — message queued")
|
||||||
|
}
|
||||||
|
// If sent, we wait for response via FCM or sync
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send transcribed text to Matrix #clippy-voice room.
|
||||||
|
*/
|
||||||
|
private suspend fun sendToMatrix(text: String): Boolean {
|
||||||
|
return withContext(Dispatchers.IO) {
|
||||||
|
matrixClient.sendMessage(
|
||||||
|
roomId = roomId,
|
||||||
|
body = text,
|
||||||
|
source = "voice",
|
||||||
|
sessionId = sessionId
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Handle Clippy's response — from either FCM or Matrix sync.
|
||||||
|
*/
|
||||||
|
private fun handleClippyResponse(text: String) {
|
||||||
|
// Dedup — mark this response so the other path skips it
|
||||||
|
val hash = sha256(text)
|
||||||
|
if (ResponseDedup.seen(hash)) return
|
||||||
|
ResponseDedup.mark(hash)
|
||||||
|
|
||||||
|
stateMachine.transition(VoiceState.RECEIVING)
|
||||||
|
onStateChange(VoiceState.RECEIVING, "")
|
||||||
|
|
||||||
|
// Send to TTS — TTSManager handles queuing if multiple chunks arrive
|
||||||
|
ttsManager.speak(text)
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Background sync loop — polls Matrix for messages.
|
||||||
|
* This is the fallback path when FCM doesn't deliver first.
|
||||||
|
*/
|
||||||
|
private fun startSyncLoop() {
|
||||||
|
syncJob = scope.launch(Dispatchers.IO) {
|
||||||
|
while (isActive) {
|
||||||
|
try {
|
||||||
|
val messages = matrixClient.sync(roomId, timeoutMs = 30000)
|
||||||
|
messages
|
||||||
|
.filter { it.sender == clippyUserId }
|
||||||
|
.filter { it.signal == null } // Skip session signals
|
||||||
|
.forEach { msg ->
|
||||||
|
withContext(Dispatchers.Main) {
|
||||||
|
handleClippyResponse(msg.body)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} catch (e: Exception) {
|
||||||
|
delay(5000) // Back off on error
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Flush messages queued during offline period.
|
||||||
|
*/
|
||||||
|
private suspend fun flushQueuedMessages() {
|
||||||
|
val queued = messageQueue.flush()
|
||||||
|
for (msg in queued) {
|
||||||
|
sendToMatrix(msg.text)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send session start signal.
|
||||||
|
*/
|
||||||
|
fun startSession() {
|
||||||
|
sessionId = Instant.now().toString()
|
||||||
|
scope.launch(Dispatchers.IO) {
|
||||||
|
matrixClient.sendSessionSignal(roomId, "session_start", sessionId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Send session end signal.
|
||||||
|
*/
|
||||||
|
fun endSession() {
|
||||||
|
scope.launch(Dispatchers.IO) {
|
||||||
|
matrixClient.sendSessionSignal(roomId, "session_end", sessionId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
fun destroy() {
|
||||||
|
syncJob?.cancel()
|
||||||
|
speechManager.destroy()
|
||||||
|
ttsManager.shutdown()
|
||||||
|
networkMonitor.stop()
|
||||||
|
scope.cancel()
|
||||||
|
}
|
||||||
|
|
||||||
|
private fun sha256(text: String): String {
|
||||||
|
val digest = MessageDigest.getInstance("SHA-256")
|
||||||
|
return digest.digest(text.toByteArray()).joinToString("") { "%02x".format(it) }
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user