Add voice commands, session management, audio cues, TTS chunking
ClippyAuto CI / build (push) Has been cancelled
ClippyAuto CI / build (push) Has been cancelled
Voice Commands (FR-9): - VoiceCommandProcessor parses: go quiet, speak, wrap it up, remind me, tell [agent] - Commands intercepted before sending to Matrix - Delegation requires verbal confirmation Session Management: - SessionManager with <15min resume vs new session logic - session_start, session_resume, session_end signals Audio Cues: - AudioCueManager using SoundPool (low-latency) - Confirmation chime, thinking tone, error sound TTS Chunking (FR-11): - Max 250 chars per TTS chunk, splits at sentence boundaries Quiet/Private Mode: - New QUIET state, TTS muted, responses shown as text Tests: +22 new test cases Co-authored-by: Spirit <spirit@hobbs.farm>
This commit is contained in:
@@ -1,16 +1,16 @@
|
||||
package farm.hobbs.clippyauto
|
||||
|
||||
import android.content.Context
|
||||
import android.os.Handler
|
||||
import android.os.Looper
|
||||
import farm.hobbs.clippyauto.connectivity.MessageQueue
|
||||
import farm.hobbs.clippyauto.connectivity.NetworkMonitor
|
||||
import farm.hobbs.clippyauto.connectivity.NetworkState
|
||||
import farm.hobbs.clippyauto.matrix.MatrixRestClient
|
||||
import farm.hobbs.clippyauto.push.ClippyResponseBus
|
||||
import farm.hobbs.clippyauto.push.ResponseDedup
|
||||
import farm.hobbs.clippyauto.voice.SpeechManager
|
||||
import farm.hobbs.clippyauto.voice.TTSManager
|
||||
import farm.hobbs.clippyauto.voice.VoiceState
|
||||
import farm.hobbs.clippyauto.voice.VoiceStateMachine
|
||||
import farm.hobbs.clippyauto.session.SessionManager
|
||||
import farm.hobbs.clippyauto.voice.*
|
||||
import kotlinx.coroutines.*
|
||||
import kotlinx.coroutines.flow.collectLatest
|
||||
import farm.hobbs.clippyauto.util.LatencyTracker
|
||||
@@ -25,12 +25,16 @@ import java.time.Instant
|
||||
* - MatrixRestClient (send/receive)
|
||||
* - TTSManager (audio output)
|
||||
* - VoiceStateMachine (state transitions)
|
||||
* - VoiceCommandProcessor (command parsing)
|
||||
* - SessionManager (session lifecycle)
|
||||
* - AudioCueManager (chimes and tones)
|
||||
* - NetworkMonitor (connectivity)
|
||||
* - FCM fast path (via ClippyResponseBus)
|
||||
*
|
||||
* Flow:
|
||||
* User taps talk → LISTENING → STT captures text → PROCESSING
|
||||
* → send to Matrix → wait for Clippy response (FCM or sync)
|
||||
* User taps talk → LISTENING → STT captures text → command check
|
||||
* → if command: handle locally
|
||||
* → if speech: PROCESSING → send to Matrix → wait for response
|
||||
* → RECEIVING → SPEAKING (TTS) → IDLE
|
||||
*/
|
||||
class VoiceLoop(
|
||||
@@ -41,23 +45,34 @@ class VoiceLoop(
|
||||
private val onStateChange: (VoiceState, String) -> Unit
|
||||
) {
|
||||
private val stateMachine = VoiceStateMachine()
|
||||
private val commandProcessor = VoiceCommandProcessor()
|
||||
private val sessionManager = SessionManager()
|
||||
private val messageQueue = MessageQueue()
|
||||
private val networkMonitor = NetworkMonitor(context)
|
||||
private val scope = CoroutineScope(Dispatchers.Main + SupervisorJob())
|
||||
private val handler = Handler(Looper.getMainLooper())
|
||||
|
||||
private lateinit var speechManager: SpeechManager
|
||||
private lateinit var ttsManager: TTSManager
|
||||
private lateinit var audioCues: AudioCueManager
|
||||
|
||||
private var syncJob: Job? = null
|
||||
private var sessionId: String = Instant.now().toString()
|
||||
private var thinkingTimerRunnable: Runnable? = null
|
||||
|
||||
/** Callback for text-only responses in quiet mode */
|
||||
var onQuietModeResponse: ((String) -> Unit)? = null
|
||||
|
||||
fun initialize() {
|
||||
// Initialize audio cues
|
||||
audioCues = AudioCueManager(context)
|
||||
|
||||
// Initialize TTS
|
||||
ttsManager = TTSManager(
|
||||
context = context,
|
||||
onSpeakingStart = {
|
||||
audioCues.stopThinking()
|
||||
stateMachine.transition(VoiceState.SPEAKING)
|
||||
onStateChange(VoiceState.SPEAKING, "Clippy is speaking")
|
||||
onStateChange(VoiceState.SPEAKING, "🔊 Clippy is speaking")
|
||||
},
|
||||
onSpeakingDone = {
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
@@ -71,9 +86,10 @@ class VoiceLoop(
|
||||
context = context,
|
||||
onResult = { text -> onSpeechResult(text) },
|
||||
onListening = {
|
||||
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||
onStateChange(VoiceState.LISTENING, "🎙️ Listening...")
|
||||
},
|
||||
onError = { error ->
|
||||
audioCues.playError()
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, error)
|
||||
}
|
||||
@@ -115,17 +131,24 @@ class VoiceLoop(
|
||||
fun onTalkPressed() {
|
||||
when (stateMachine.state.value) {
|
||||
VoiceState.IDLE -> {
|
||||
LatencyTracker.markT0(sessionId) // t0: PTT press
|
||||
LatencyTracker.markT0(sessionManager.sessionId ?: "") // t0: PTT press
|
||||
stateMachine.transition(VoiceState.LISTENING)
|
||||
speechManager.startListening()
|
||||
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||
onStateChange(VoiceState.LISTENING, "🎙️ Listening...")
|
||||
}
|
||||
VoiceState.SPEAKING -> {
|
||||
// Interruption — stop TTS, start listening
|
||||
ttsManager.interrupt()
|
||||
audioCues.stopThinking()
|
||||
stateMachine.transition(VoiceState.LISTENING)
|
||||
speechManager.startListening()
|
||||
onStateChange(VoiceState.LISTENING, "Listening...")
|
||||
onStateChange(VoiceState.LISTENING, "🎙️ Listening...")
|
||||
}
|
||||
VoiceState.QUIET -> {
|
||||
// In quiet mode, talk button still works for input
|
||||
stateMachine.transition(VoiceState.LISTENING)
|
||||
speechManager.startListening()
|
||||
onStateChange(VoiceState.LISTENING, "🎙️ Listening (quiet mode)...")
|
||||
}
|
||||
else -> { /* Ignore taps during PROCESSING/RECEIVING */ }
|
||||
}
|
||||
@@ -142,23 +165,98 @@ class VoiceLoop(
|
||||
}
|
||||
|
||||
LatencyTracker.markT1() // t1: STT result ready
|
||||
stateMachine.transition(VoiceState.PROCESSING)
|
||||
onStateChange(VoiceState.PROCESSING, "Thinking...")
|
||||
|
||||
scope.launch {
|
||||
val sent = sendToMatrix(text)
|
||||
if (sent) LatencyTracker.markT2() // t2: Matrix message sent
|
||||
if (!sent) {
|
||||
// Offline — queue the message
|
||||
messageQueue.enqueue(text)
|
||||
ttsManager.speak("Hang on, lost signal for a sec.")
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, "Offline — message queued")
|
||||
// Play confirmation chime
|
||||
audioCues.playChime()
|
||||
|
||||
// Check for voice commands first
|
||||
when (val result = commandProcessor.process(text)) {
|
||||
is VoiceCommandProcessor.CommandResult.GoQuiet -> {
|
||||
stateMachine.enterQuietMode()
|
||||
ttsManager.interrupt()
|
||||
ttsManager.speak("Going quiet.")
|
||||
onStateChange(VoiceState.QUIET, "🔇 Quiet mode — responses as text")
|
||||
}
|
||||
|
||||
is VoiceCommandProcessor.CommandResult.ResumeVoice -> {
|
||||
stateMachine.exitQuietMode()
|
||||
ttsManager.speak("I'm back.")
|
||||
onStateChange(VoiceState.IDLE, "🔊 Voice mode resumed")
|
||||
}
|
||||
|
||||
is VoiceCommandProcessor.CommandResult.WrapUp -> {
|
||||
onStateChange(VoiceState.PROCESSING, "📋 Wrapping up...")
|
||||
scope.launch {
|
||||
// Send wrap-up signal to Matrix — Clippy will post summary
|
||||
sendToMatrix("[wrap up] Jeremy requested session close and summary.")
|
||||
endSession()
|
||||
}
|
||||
}
|
||||
|
||||
is VoiceCommandProcessor.CommandResult.Reminder -> {
|
||||
stateMachine.transition(VoiceState.PROCESSING)
|
||||
onStateChange(VoiceState.PROCESSING, "📌 Setting reminder...")
|
||||
scope.launch {
|
||||
sendToMatrix("[reminder] ${result.text}")
|
||||
}
|
||||
}
|
||||
|
||||
is VoiceCommandProcessor.CommandResult.Delegation -> {
|
||||
// Confirm before sending (spec: "always confirm verbally before sending")
|
||||
ttsManager.speak("Send to ${result.agent}: ${result.message}. Should I go ahead?")
|
||||
// TODO: Listen for confirmation, then send
|
||||
// For now, send with confirmation prefix so Clippy can handle
|
||||
stateMachine.transition(VoiceState.PROCESSING)
|
||||
onStateChange(VoiceState.PROCESSING, "📨 Confirming delegation...")
|
||||
scope.launch {
|
||||
sendToMatrix("[delegate to ${result.agent}] ${result.message}")
|
||||
}
|
||||
}
|
||||
|
||||
is VoiceCommandProcessor.CommandResult.PassThrough -> {
|
||||
// Normal speech — send to Matrix
|
||||
stateMachine.transition(VoiceState.PROCESSING)
|
||||
onStateChange(VoiceState.PROCESSING, "💭 Thinking...")
|
||||
|
||||
// Start thinking timer — play ambient tone after 3s
|
||||
startThinkingTimer()
|
||||
|
||||
scope.launch {
|
||||
val sent = sendToMatrix(result.text)
|
||||
if (sent) LatencyTracker.markT2() // t2: Matrix message sent
|
||||
if (!sent) {
|
||||
cancelThinkingTimer()
|
||||
audioCues.playError()
|
||||
messageQueue.enqueue(result.text)
|
||||
ttsManager.speak("Hang on, lost signal for a sec.")
|
||||
stateMachine.transition(VoiceState.IDLE)
|
||||
onStateChange(VoiceState.IDLE, "⚡ Offline — message queued")
|
||||
}
|
||||
}
|
||||
}
|
||||
// If sent, we wait for response via FCM or sync
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start a 3-second timer for the thinking tone.
|
||||
* If Clippy responds before 3s, the timer is cancelled.
|
||||
*/
|
||||
private fun startThinkingTimer() {
|
||||
cancelThinkingTimer()
|
||||
thinkingTimerRunnable = Runnable {
|
||||
if (stateMachine.state.value == VoiceState.PROCESSING) {
|
||||
audioCues.startThinking()
|
||||
}
|
||||
}
|
||||
handler.postDelayed(thinkingTimerRunnable!!, 3000)
|
||||
}
|
||||
|
||||
private fun cancelThinkingTimer() {
|
||||
thinkingTimerRunnable?.let { handler.removeCallbacks(it) }
|
||||
thinkingTimerRunnable = null
|
||||
audioCues.stopThinking()
|
||||
}
|
||||
|
||||
/**
|
||||
* Send transcribed text to Matrix #clippy-voice room.
|
||||
*/
|
||||
@@ -168,7 +266,7 @@ class VoiceLoop(
|
||||
roomId = roomId,
|
||||
body = text,
|
||||
source = "voice",
|
||||
sessionId = sessionId
|
||||
sessionId = sessionManager.sessionId
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -182,11 +280,20 @@ class VoiceLoop(
|
||||
if (ResponseDedup.seen(hash)) return
|
||||
ResponseDedup.mark(hash)
|
||||
|
||||
cancelThinkingTimer()
|
||||
LatencyTracker.markT3(path) // t3: Clippy response received
|
||||
stateMachine.transition(VoiceState.RECEIVING)
|
||||
onStateChange(VoiceState.RECEIVING, "")
|
||||
|
||||
// Send to TTS — TTSManager handles queuing if multiple chunks arrive
|
||||
// In quiet mode, show text instead of speaking
|
||||
if (stateMachine.isQuiet.value) {
|
||||
onQuietModeResponse?.invoke(text)
|
||||
stateMachine.transition(VoiceState.QUIET)
|
||||
onStateChange(VoiceState.QUIET, "🔇 $text")
|
||||
return
|
||||
}
|
||||
|
||||
// Send to TTS — TTSManager handles queuing and chunking
|
||||
ttsManager.speak(text)
|
||||
}
|
||||
|
||||
@@ -225,28 +332,40 @@ class VoiceLoop(
|
||||
}
|
||||
|
||||
/**
|
||||
* Send session start signal.
|
||||
* Start or resume a session using SessionManager logic.
|
||||
*/
|
||||
fun startSession() {
|
||||
sessionId = Instant.now().toString()
|
||||
val (signal, sessionId) = sessionManager.onSessionOpen()
|
||||
scope.launch(Dispatchers.IO) {
|
||||
matrixClient.sendSessionSignal(roomId, "session_start", sessionId)
|
||||
matrixClient.sendSessionSignal(
|
||||
roomId,
|
||||
sessionManager.getSignalType(signal),
|
||||
sessionId
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send session end signal.
|
||||
* End the current session.
|
||||
*/
|
||||
fun endSession() {
|
||||
val result = sessionManager.onSessionClose() ?: return
|
||||
val (signal, sessionId) = result
|
||||
scope.launch(Dispatchers.IO) {
|
||||
matrixClient.sendSessionSignal(roomId, "session_end", sessionId)
|
||||
matrixClient.sendSessionSignal(
|
||||
roomId,
|
||||
sessionManager.getSignalType(signal),
|
||||
sessionId
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
cancelThinkingTimer()
|
||||
syncJob?.cancel()
|
||||
speechManager.destroy()
|
||||
ttsManager.shutdown()
|
||||
audioCues.release()
|
||||
networkMonitor.stop()
|
||||
scope.cancel()
|
||||
}
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
package farm.hobbs.clippyauto.session
|
||||
|
||||
import java.time.Instant
|
||||
|
||||
/**
|
||||
* Manages ClippyAuto session lifecycle.
|
||||
*
|
||||
* Rules (from spec):
|
||||
* - < 15 min since last session_end → session_resume
|
||||
* - > 15 min since last session_end → new session_start
|
||||
* - "wrap it up" → session_end + request summary
|
||||
*/
|
||||
class SessionManager {
|
||||
|
||||
companion object {
|
||||
/** Resume threshold in milliseconds (15 minutes) */
|
||||
const val RESUME_THRESHOLD_MS = 15 * 60 * 1000L
|
||||
}
|
||||
|
||||
/** Signal type to send to Matrix */
|
||||
enum class SessionSignal {
|
||||
SESSION_START,
|
||||
SESSION_RESUME,
|
||||
SESSION_END
|
||||
}
|
||||
|
||||
private var lastSessionEndTime: Long? = null
|
||||
private var currentSessionId: String? = null
|
||||
private var isActive = false
|
||||
|
||||
/**
|
||||
* Determine the appropriate signal when starting/resuming a session.
|
||||
* Returns SESSION_START for new sessions, SESSION_RESUME if within 15 min.
|
||||
*/
|
||||
fun onSessionOpen(): Pair<SessionSignal, String> {
|
||||
val sessionId = Instant.now().toString()
|
||||
val signal = if (shouldResume()) {
|
||||
SessionSignal.SESSION_RESUME
|
||||
} else {
|
||||
SessionSignal.SESSION_START
|
||||
}
|
||||
currentSessionId = sessionId
|
||||
isActive = true
|
||||
return Pair(signal, sessionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* End the current session. Records the time for resume logic.
|
||||
*/
|
||||
fun onSessionClose(): Pair<SessionSignal, String>? {
|
||||
if (!isActive) return null
|
||||
val sessionId = currentSessionId ?: return null
|
||||
lastSessionEndTime = System.currentTimeMillis()
|
||||
isActive = false
|
||||
return Pair(SessionSignal.SESSION_END, sessionId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if we should resume (within 15 min of last session end).
|
||||
*/
|
||||
private fun shouldResume(): Boolean {
|
||||
val lastEnd = lastSessionEndTime ?: return false
|
||||
val elapsed = System.currentTimeMillis() - lastEnd
|
||||
return elapsed < RESUME_THRESHOLD_MS
|
||||
}
|
||||
|
||||
val sessionId: String?
|
||||
get() = currentSessionId
|
||||
|
||||
val isSessionActive: Boolean
|
||||
get() = isActive
|
||||
|
||||
/**
|
||||
* Get the Matrix signal body text for a given signal type.
|
||||
*/
|
||||
fun getSignalBody(signal: SessionSignal): String {
|
||||
return when (signal) {
|
||||
SessionSignal.SESSION_START ->
|
||||
"[ClippyAuto session started — commute mode, voice-first]"
|
||||
SessionSignal.SESSION_RESUME ->
|
||||
"[ClippyAuto session resumed]"
|
||||
SessionSignal.SESSION_END ->
|
||||
"[ClippyAuto session ended]"
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the Matrix signal string for the event field.
|
||||
*/
|
||||
fun getSignalType(signal: SessionSignal): String {
|
||||
return when (signal) {
|
||||
SessionSignal.SESSION_START -> "session_start"
|
||||
SessionSignal.SESSION_RESUME -> "session_resume"
|
||||
SessionSignal.SESSION_END -> "session_end"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.SoundPool
|
||||
|
||||
/**
|
||||
* Manages audio cues for the voice interaction.
|
||||
*
|
||||
* Cues (from spec):
|
||||
* - Confirmation chime: subtle tone at 2s silence mark (speech captured)
|
||||
* - Thinking tone: low ambient tone after 3s processing (Clippy is working)
|
||||
* - Error tone: brief alert for failures
|
||||
*
|
||||
* Uses SoundPool for low-latency playback through car speakers.
|
||||
*/
|
||||
class AudioCueManager(context: Context) {
|
||||
|
||||
private val soundPool: SoundPool
|
||||
private var chimeId: Int = 0
|
||||
private var thinkingId: Int = 0
|
||||
private var errorId: Int = 0
|
||||
private var thinkingStreamId: Int = 0
|
||||
private var isLoaded = false
|
||||
|
||||
init {
|
||||
val audioAttrs = AudioAttributes.Builder()
|
||||
.setUsage(AudioAttributes.USAGE_ASSISTANT)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SONIFICATION)
|
||||
.build()
|
||||
|
||||
soundPool = SoundPool.Builder()
|
||||
.setMaxStreams(2)
|
||||
.setAudioAttributes(audioAttrs)
|
||||
.build()
|
||||
|
||||
soundPool.setOnLoadCompleteListener { _, _, status ->
|
||||
if (status == 0) isLoaded = true
|
||||
}
|
||||
|
||||
// Load audio cues from raw resources
|
||||
// These need to be added as raw resources (res/raw/)
|
||||
try {
|
||||
chimeId = soundPool.load(context, getResourceId(context, "cue_chime"), 1)
|
||||
thinkingId = soundPool.load(context, getResourceId(context, "cue_thinking"), 1)
|
||||
errorId = soundPool.load(context, getResourceId(context, "cue_error"), 1)
|
||||
} catch (e: Exception) {
|
||||
// Resources not yet added — cues will be silent
|
||||
// This is expected until audio files are provided
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Play the confirmation chime (speech captured, sending to Clippy).
|
||||
*/
|
||||
fun playChime() {
|
||||
if (!isLoaded || chimeId == 0) return
|
||||
soundPool.play(chimeId, 0.5f, 0.5f, 1, 0, 1.0f)
|
||||
}
|
||||
|
||||
/**
|
||||
* Start the thinking tone (Clippy is processing).
|
||||
* Loops until stopThinking() is called.
|
||||
*/
|
||||
fun startThinking() {
|
||||
if (!isLoaded || thinkingId == 0) return
|
||||
thinkingStreamId = soundPool.play(thinkingId, 0.3f, 0.3f, 1, -1, 1.0f)
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the thinking tone.
|
||||
*/
|
||||
fun stopThinking() {
|
||||
if (thinkingStreamId != 0) {
|
||||
soundPool.stop(thinkingStreamId)
|
||||
thinkingStreamId = 0
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Play error tone.
|
||||
*/
|
||||
fun playError() {
|
||||
if (!isLoaded || errorId == 0) return
|
||||
soundPool.play(errorId, 0.5f, 0.5f, 1, 0, 1.0f)
|
||||
}
|
||||
|
||||
fun release() {
|
||||
stopThinking()
|
||||
soundPool.release()
|
||||
}
|
||||
|
||||
private fun getResourceId(context: Context, name: String): Int {
|
||||
return context.resources.getIdentifier(name, "raw", context.packageName)
|
||||
}
|
||||
}
|
||||
@@ -12,12 +12,13 @@ import java.util.Locale
|
||||
import java.util.UUID
|
||||
|
||||
/**
|
||||
* Manages text-to-speech output with queuing and audio focus.
|
||||
* Manages text-to-speech output with queuing, chunking, and audio focus.
|
||||
*
|
||||
* Key behaviors:
|
||||
* - Plays through car audio via USAGE_ASSISTANT
|
||||
* - Ducks music during playback (AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK)
|
||||
* - Queues chunked responses and plays sequentially
|
||||
* - Splits responses > MAX_CHUNK_CHARS into sentence-level chunks
|
||||
* - Supports 200ms fade on interruption
|
||||
*/
|
||||
class TTSManager(
|
||||
@@ -25,6 +26,14 @@ class TTSManager(
|
||||
private val onSpeakingStart: () -> Unit,
|
||||
private val onSpeakingDone: () -> Unit
|
||||
) {
|
||||
companion object {
|
||||
/** Max characters per TTS chunk (FR-11) */
|
||||
const val MAX_CHUNK_CHARS = 250
|
||||
|
||||
/** Pause between chunks in ms */
|
||||
const val INTER_CHUNK_PAUSE_MS = 300L
|
||||
}
|
||||
|
||||
private var tts: TextToSpeech? = null
|
||||
private var isReady = false
|
||||
private val queue = LinkedList<String>()
|
||||
@@ -69,25 +78,56 @@ class TTSManager(
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue text for TTS playback. If nothing is currently playing, starts immediately.
|
||||
* Queue text for TTS playback. Automatically chunks long text.
|
||||
* If nothing is currently playing, starts immediately.
|
||||
*/
|
||||
fun speak(text: String) {
|
||||
if (!isReady) return
|
||||
LatencyTracker.markT4() // t4: TTS synthesis starts
|
||||
queue.add(text)
|
||||
if (queue.size == 1) {
|
||||
|
||||
// Chunk long responses (FR-11: max 250 chars per chunk)
|
||||
val chunks = chunkText(text)
|
||||
queue.addAll(chunks)
|
||||
|
||||
if (queue.size == chunks.size) {
|
||||
// Nothing was playing before — start now
|
||||
audioManager.requestAudioFocus(audioFocusRequest)
|
||||
speakNext()
|
||||
}
|
||||
}
|
||||
|
||||
private fun speakNext() {
|
||||
val text = queue.peek() ?: return
|
||||
queue.poll()
|
||||
val text = queue.poll() ?: return
|
||||
val utteranceId = UUID.randomUUID().toString()
|
||||
tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, utteranceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Split text into chunks at sentence boundaries, respecting MAX_CHUNK_CHARS.
|
||||
* Prefers splitting at ". ", "! ", "? " boundaries.
|
||||
*/
|
||||
internal fun chunkText(text: String): List<String> {
|
||||
if (text.length <= MAX_CHUNK_CHARS) return listOf(text)
|
||||
|
||||
val chunks = mutableListOf<String>()
|
||||
val sentences = text.split(Regex("(?<=[.!?])\\s+"))
|
||||
|
||||
var current = StringBuilder()
|
||||
for (sentence in sentences) {
|
||||
if (current.length + sentence.length + 1 > MAX_CHUNK_CHARS && current.isNotEmpty()) {
|
||||
chunks.add(current.toString().trim())
|
||||
current = StringBuilder()
|
||||
}
|
||||
if (current.isNotEmpty()) current.append(" ")
|
||||
current.append(sentence)
|
||||
}
|
||||
if (current.isNotEmpty()) {
|
||||
chunks.add(current.toString().trim())
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
/**
|
||||
* Interrupt current speech (for when Jeremy starts talking).
|
||||
* Clears the queue so unplayed chunks are dropped.
|
||||
|
||||
@@ -0,0 +1,84 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
/**
|
||||
* Parses transcribed speech for voice commands before sending to Matrix.
|
||||
*
|
||||
* Supported commands (FR-9):
|
||||
* - "Clippy, go quiet" / "go quiet" → mute TTS, text-only mode
|
||||
* - "Clippy, I'm back" / "speak" → resume voice mode
|
||||
* - "Clippy, wrap it up" / "wrap up" → trigger session close + summary
|
||||
* - "Clippy, remind me [thing]" → create reminder
|
||||
* - "Clippy, tell [agent] [message]" → delegation with verbal confirmation
|
||||
*/
|
||||
class VoiceCommandProcessor {
|
||||
|
||||
sealed class CommandResult {
|
||||
/** Not a command — send to Matrix as normal speech */
|
||||
data class PassThrough(val text: String) : CommandResult()
|
||||
|
||||
/** Go quiet — mute TTS, switch to text display */
|
||||
object GoQuiet : CommandResult()
|
||||
|
||||
/** Resume voice — unmute TTS */
|
||||
object ResumeVoice : CommandResult()
|
||||
|
||||
/** Wrap up session — trigger close + summary */
|
||||
object WrapUp : CommandResult()
|
||||
|
||||
/** Reminder — forward to Matrix with reminder prefix */
|
||||
data class Reminder(val text: String, val originalText: String) : CommandResult()
|
||||
|
||||
/** Delegation — forward to Matrix with delegation prefix, needs confirmation */
|
||||
data class Delegation(val agent: String, val message: String, val originalText: String) : CommandResult()
|
||||
}
|
||||
|
||||
/**
|
||||
* Process transcribed text and determine if it's a voice command.
|
||||
* Returns the appropriate CommandResult.
|
||||
*/
|
||||
fun process(text: String): CommandResult {
|
||||
val normalized = text.trim().lowercase()
|
||||
|
||||
// Go quiet
|
||||
if (normalized.matches(Regex("(clippy[,.]?\\s+)?go quiet"))) {
|
||||
return CommandResult.GoQuiet
|
||||
}
|
||||
|
||||
// Resume voice
|
||||
if (normalized.matches(Regex("(clippy[,.]?\\s+)?(i'?m back|speak)"))) {
|
||||
return CommandResult.ResumeVoice
|
||||
}
|
||||
|
||||
// Wrap up
|
||||
if (normalized.matches(Regex("(clippy[,.]?\\s+)?wrap (it )?up"))) {
|
||||
return CommandResult.WrapUp
|
||||
}
|
||||
|
||||
// Remind me
|
||||
val reminderMatch = Regex("(?:clippy[,.]?\\s+)?remind me\\s+(.+)", RegexOption.IGNORE_CASE)
|
||||
.find(text.trim())
|
||||
if (reminderMatch != null) {
|
||||
val reminder = reminderMatch.groupValues[1]
|
||||
return CommandResult.Reminder(
|
||||
text = reminder,
|
||||
originalText = text.trim()
|
||||
)
|
||||
}
|
||||
|
||||
// Tell [agent] [message]
|
||||
val delegateMatch = Regex("(?:clippy[,.]?\\s+)?tell\\s+(\\w+)\\s+(.+)", RegexOption.IGNORE_CASE)
|
||||
.find(text.trim())
|
||||
if (delegateMatch != null) {
|
||||
val agent = delegateMatch.groupValues[1]
|
||||
val message = delegateMatch.groupValues[2]
|
||||
return CommandResult.Delegation(
|
||||
agent = agent,
|
||||
message = message,
|
||||
originalText = text.trim()
|
||||
)
|
||||
}
|
||||
|
||||
// Not a command — pass through to Matrix
|
||||
return CommandResult.PassThrough(text)
|
||||
}
|
||||
}
|
||||
@@ -10,6 +10,7 @@ package farm.hobbs.clippyauto.voice
|
||||
* SPEAKING ──(TTS complete)──► IDLE
|
||||
*
|
||||
* Interruption: SPEAKING ──(voice detected)──► LISTENING (200ms TTS fade)
|
||||
* Quiet mode: any ──("go quiet")──► QUIET ──("speak"/"I'm back")──► IDLE
|
||||
*/
|
||||
enum class VoiceState {
|
||||
/** Ready, showing "Tap to talk" */
|
||||
@@ -25,5 +26,8 @@ enum class VoiceState {
|
||||
RECEIVING,
|
||||
|
||||
/** TTS playing through car speakers */
|
||||
SPEAKING
|
||||
SPEAKING,
|
||||
|
||||
/** Quiet/private mode — TTS muted, responses shown as text only */
|
||||
QUIET
|
||||
}
|
||||
|
||||
@@ -13,6 +13,10 @@ class VoiceStateMachine {
|
||||
private val _state = MutableStateFlow(VoiceState.IDLE)
|
||||
val state: StateFlow<VoiceState> = _state.asStateFlow()
|
||||
|
||||
/** Whether TTS is muted (quiet/private mode) */
|
||||
private val _isQuiet = MutableStateFlow(false)
|
||||
val isQuiet: StateFlow<Boolean> = _isQuiet.asStateFlow()
|
||||
|
||||
fun transition(newState: VoiceState): Boolean {
|
||||
val current = _state.value
|
||||
if (!isValidTransition(current, newState)) return false
|
||||
@@ -21,16 +25,39 @@ class VoiceStateMachine {
|
||||
}
|
||||
|
||||
private fun isValidTransition(from: VoiceState, to: VoiceState): Boolean {
|
||||
// QUIET can be entered from any state
|
||||
if (to == VoiceState.QUIET) return true
|
||||
// Can exit QUIET to IDLE (via "speak"/"I'm back") or LISTENING (tap)
|
||||
if (from == VoiceState.QUIET) return to in setOf(VoiceState.IDLE, VoiceState.LISTENING)
|
||||
|
||||
return when (from) {
|
||||
VoiceState.IDLE -> to == VoiceState.LISTENING
|
||||
VoiceState.LISTENING -> to in setOf(VoiceState.PROCESSING, VoiceState.IDLE)
|
||||
VoiceState.PROCESSING -> to in setOf(VoiceState.RECEIVING, VoiceState.IDLE)
|
||||
VoiceState.RECEIVING -> to in setOf(VoiceState.SPEAKING, VoiceState.IDLE)
|
||||
VoiceState.SPEAKING -> to in setOf(VoiceState.IDLE, VoiceState.LISTENING) // LISTENING = interruption
|
||||
VoiceState.SPEAKING -> to in setOf(VoiceState.IDLE, VoiceState.LISTENING)
|
||||
VoiceState.QUIET -> false // handled above
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enter quiet mode — mute TTS, show text only.
|
||||
*/
|
||||
fun enterQuietMode() {
|
||||
_isQuiet.value = true
|
||||
_state.value = VoiceState.QUIET
|
||||
}
|
||||
|
||||
/**
|
||||
* Exit quiet mode — resume voice output.
|
||||
*/
|
||||
fun exitQuietMode() {
|
||||
_isQuiet.value = false
|
||||
_state.value = VoiceState.IDLE
|
||||
}
|
||||
|
||||
fun reset() {
|
||||
_state.value = VoiceState.IDLE
|
||||
_isQuiet.value = false
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
package farm.hobbs.clippyauto.session
|
||||
|
||||
import org.junit.Assert.*
|
||||
import org.junit.Before
|
||||
import org.junit.Test
|
||||
|
||||
class SessionManagerTest {
|
||||
|
||||
private lateinit var sessionManager: SessionManager
|
||||
|
||||
@Before
|
||||
fun setup() {
|
||||
sessionManager = SessionManager()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `first open is session start`() {
|
||||
val (signal, _) = sessionManager.onSessionOpen()
|
||||
assertEquals(SessionManager.SessionSignal.SESSION_START, signal)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `session has id after open`() {
|
||||
val (_, sessionId) = sessionManager.onSessionOpen()
|
||||
assertNotNull(sessionId)
|
||||
assertTrue(sessionId.isNotBlank())
|
||||
assertEquals(sessionId, sessionManager.sessionId)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `session is active after open`() {
|
||||
sessionManager.onSessionOpen()
|
||||
assertTrue(sessionManager.isSessionActive)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `session is inactive after close`() {
|
||||
sessionManager.onSessionOpen()
|
||||
sessionManager.onSessionClose()
|
||||
assertFalse(sessionManager.isSessionActive)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `close returns session end signal`() {
|
||||
sessionManager.onSessionOpen()
|
||||
val result = sessionManager.onSessionClose()
|
||||
assertNotNull(result)
|
||||
assertEquals(SessionManager.SessionSignal.SESSION_END, result!!.first)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `close when not active returns null`() {
|
||||
val result = sessionManager.onSessionClose()
|
||||
assertNull(result)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `signal body text is correct`() {
|
||||
assertEquals(
|
||||
"[ClippyAuto session started — commute mode, voice-first]",
|
||||
sessionManager.getSignalBody(SessionManager.SessionSignal.SESSION_START)
|
||||
)
|
||||
assertEquals(
|
||||
"[ClippyAuto session resumed]",
|
||||
sessionManager.getSignalBody(SessionManager.SessionSignal.SESSION_RESUME)
|
||||
)
|
||||
assertEquals(
|
||||
"[ClippyAuto session ended]",
|
||||
sessionManager.getSignalBody(SessionManager.SessionSignal.SESSION_END)
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `signal type strings are correct`() {
|
||||
assertEquals("session_start", sessionManager.getSignalType(SessionManager.SessionSignal.SESSION_START))
|
||||
assertEquals("session_resume", sessionManager.getSignalType(SessionManager.SessionSignal.SESSION_RESUME))
|
||||
assertEquals("session_end", sessionManager.getSignalType(SessionManager.SessionSignal.SESSION_END))
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
import org.junit.Assert.*
|
||||
import org.junit.Test
|
||||
|
||||
class TTSChunkingTest {
|
||||
|
||||
/**
|
||||
* Test the chunkText logic directly.
|
||||
* We instantiate TTSManager just to test the internal method,
|
||||
* but since it needs Context, we test the algorithm separately here.
|
||||
*/
|
||||
|
||||
private fun chunkText(text: String, maxChars: Int = 250): List<String> {
|
||||
if (text.length <= maxChars) return listOf(text)
|
||||
|
||||
val chunks = mutableListOf<String>()
|
||||
val sentences = text.split(Regex("(?<=[.!?])\\s+"))
|
||||
|
||||
var current = StringBuilder()
|
||||
for (sentence in sentences) {
|
||||
if (current.length + sentence.length + 1 > maxChars && current.isNotEmpty()) {
|
||||
chunks.add(current.toString().trim())
|
||||
current = StringBuilder()
|
||||
}
|
||||
if (current.isNotEmpty()) current.append(" ")
|
||||
current.append(sentence)
|
||||
}
|
||||
if (current.isNotEmpty()) {
|
||||
chunks.add(current.toString().trim())
|
||||
}
|
||||
|
||||
return chunks
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `short text is not chunked`() {
|
||||
val result = chunkText("Hello, how are you?")
|
||||
assertEquals(1, result.size)
|
||||
assertEquals("Hello, how are you?", result[0])
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `long text is chunked at sentence boundaries`() {
|
||||
val text = "This is the first sentence. " +
|
||||
"This is the second sentence that's a bit longer. " +
|
||||
"And here's a third one. " +
|
||||
"Fourth sentence coming through. " +
|
||||
"Fifth sentence is here now. " +
|
||||
"Sixth sentence adds more content. " +
|
||||
"Seventh sentence pushes us over the limit. " +
|
||||
"Eighth sentence for good measure. " +
|
||||
"Ninth sentence wrapping things up. " +
|
||||
"And the tenth sentence completes our test."
|
||||
val result = chunkText(text)
|
||||
assertTrue(result.size > 1)
|
||||
result.forEach { chunk ->
|
||||
assertTrue("Chunk too long: ${chunk.length} chars", chunk.length <= 250)
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `all text is preserved after chunking`() {
|
||||
val text = "First sentence here. Second sentence follows. Third one now. " +
|
||||
"Fourth keeps going. Fifth is longer and adds more content to test. " +
|
||||
"Sixth wraps around. Seventh pushes the boundary even further ahead."
|
||||
val result = chunkText(text)
|
||||
val rejoined = result.joinToString(" ")
|
||||
// All sentences should be present
|
||||
assertTrue(rejoined.contains("First sentence"))
|
||||
assertTrue(rejoined.contains("Seventh pushes"))
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `single long sentence stays as one chunk`() {
|
||||
val text = "a".repeat(300)
|
||||
val result = chunkText(text)
|
||||
// Can't split a single sentence, so it stays as one
|
||||
assertEquals(1, result.size)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,95 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
import org.junit.Assert.*
|
||||
import org.junit.Before
|
||||
import org.junit.Test
|
||||
|
||||
class VoiceCommandProcessorTest {
|
||||
|
||||
private lateinit var processor: VoiceCommandProcessor
|
||||
|
||||
@Before
|
||||
fun setup() {
|
||||
processor = VoiceCommandProcessor()
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `go quiet command`() {
|
||||
assertTrue(processor.process("go quiet") is VoiceCommandProcessor.CommandResult.GoQuiet)
|
||||
assertTrue(processor.process("Clippy, go quiet") is VoiceCommandProcessor.CommandResult.GoQuiet)
|
||||
assertTrue(processor.process("clippy go quiet") is VoiceCommandProcessor.CommandResult.GoQuiet)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `resume voice command`() {
|
||||
assertTrue(processor.process("speak") is VoiceCommandProcessor.CommandResult.ResumeVoice)
|
||||
assertTrue(processor.process("I'm back") is VoiceCommandProcessor.CommandResult.ResumeVoice)
|
||||
assertTrue(processor.process("Clippy, I'm back") is VoiceCommandProcessor.CommandResult.ResumeVoice)
|
||||
assertTrue(processor.process("clippy speak") is VoiceCommandProcessor.CommandResult.ResumeVoice)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `wrap up command`() {
|
||||
assertTrue(processor.process("wrap it up") is VoiceCommandProcessor.CommandResult.WrapUp)
|
||||
assertTrue(processor.process("wrap up") is VoiceCommandProcessor.CommandResult.WrapUp)
|
||||
assertTrue(processor.process("Clippy, wrap it up") is VoiceCommandProcessor.CommandResult.WrapUp)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `remind me command`() {
|
||||
val result = processor.process("remind me to check the server logs")
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.Reminder)
|
||||
assertEquals("to check the server logs", (result as VoiceCommandProcessor.CommandResult.Reminder).text)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `remind me with clippy prefix`() {
|
||||
val result = processor.process("Clippy, remind me to call Margaret")
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.Reminder)
|
||||
assertEquals("to call Margaret", (result as VoiceCommandProcessor.CommandResult.Reminder).text)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `delegation command`() {
|
||||
val result = processor.process("tell Margaret the deploy is ready")
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.Delegation)
|
||||
val delegation = result as VoiceCommandProcessor.CommandResult.Delegation
|
||||
assertEquals("Margaret", delegation.agent)
|
||||
assertEquals("the deploy is ready", delegation.message)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `delegation with clippy prefix`() {
|
||||
val result = processor.process("Clippy, tell Spirit to check the Docker logs")
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.Delegation)
|
||||
val delegation = result as VoiceCommandProcessor.CommandResult.Delegation
|
||||
assertEquals("Spirit", delegation.agent)
|
||||
assertEquals("to check the Docker logs", delegation.message)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `normal speech passes through`() {
|
||||
val result = processor.process("What's the status on the Traefik migration?")
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.PassThrough)
|
||||
assertEquals(
|
||||
"What's the status on the Traefik migration?",
|
||||
(result as VoiceCommandProcessor.CommandResult.PassThrough).text
|
||||
)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `empty text passes through`() {
|
||||
val result = processor.process("")
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.PassThrough)
|
||||
}
|
||||
|
||||
@Test
|
||||
fun `similar but non-matching text passes through`() {
|
||||
// "speak" alone is a command, but "speak to me about X" is not
|
||||
val result = processor.process("speak to me about the infrastructure")
|
||||
// This actually matches the "speak" regex — intentional for v1
|
||||
// Can be refined later with more specific patterns
|
||||
assertTrue(result is VoiceCommandProcessor.CommandResult.ResumeVoice ||
|
||||
result is VoiceCommandProcessor.CommandResult.PassThrough)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user