feat: initial project scaffold
- Android Auto CarAppService, Session, Screen - Voice pipeline: STT (SpeechManager), TTS (TTSManager), VoiceStateMachine - Matrix REST client (login, send, sync) - no E2EE, raw API - FCM push receiver with response dedup - Network monitor + offline message queue - Phone setup activity stub - Full architecture spec in docs/ARCHITECTURE.md
This commit is contained in:
@@ -0,0 +1,66 @@
|
||||
plugins {
|
||||
id("com.android.application")
|
||||
id("org.jetbrains.kotlin.android")
|
||||
id("com.google.gms.google-services")
|
||||
}
|
||||
|
||||
android {
|
||||
namespace = "farm.hobbs.clippyauto"
|
||||
compileSdk = 35
|
||||
|
||||
defaultConfig {
|
||||
applicationId = "farm.hobbs.clippyauto"
|
||||
minSdk = 28
|
||||
targetSdk = 35
|
||||
versionCode = 1
|
||||
versionName = "0.1.0-mvp"
|
||||
}
|
||||
|
||||
buildTypes {
|
||||
release {
|
||||
isMinifyEnabled = false
|
||||
proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro")
|
||||
}
|
||||
}
|
||||
|
||||
compileOptions {
|
||||
sourceCompatibility = JavaVersion.VERSION_17
|
||||
targetCompatibility = JavaVersion.VERSION_17
|
||||
}
|
||||
|
||||
kotlinOptions {
|
||||
jvmTarget = "17"
|
||||
}
|
||||
}
|
||||
|
||||
dependencies {
|
||||
// Android Auto
|
||||
implementation("androidx.car.app:app:1.7.0")
|
||||
implementation("androidx.car.app:app-projected:1.7.0")
|
||||
|
||||
// Firebase Cloud Messaging
|
||||
implementation(platform("com.google.firebase:firebase-bom:33.9.0"))
|
||||
implementation("com.google.firebase:firebase-messaging-ktx")
|
||||
|
||||
// HTTP client for Matrix REST API
|
||||
implementation("io.ktor:ktor-client-okhttp:3.0.3")
|
||||
implementation("io.ktor:ktor-client-content-negotiation:3.0.3")
|
||||
implementation("io.ktor:ktor-serialization-kotlinx-json:3.0.3")
|
||||
|
||||
// Coroutines
|
||||
implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0")
|
||||
|
||||
// Lifecycle
|
||||
implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.7")
|
||||
implementation("androidx.lifecycle:lifecycle-service:2.8.7")
|
||||
|
||||
// DataStore (settings)
|
||||
implementation("androidx.datastore:datastore-preferences:1.1.3")
|
||||
|
||||
// Security (Keystore wrapper)
|
||||
implementation("androidx.security:security-crypto:1.1.0-alpha06")
|
||||
|
||||
// Core
|
||||
implementation("androidx.core:core-ktx:1.15.0")
|
||||
implementation("androidx.appcompat:appcompat:1.7.0")
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
|
||||
package="farm.hobbs.clippyauto">
|
||||
|
||||
<uses-permission android:name="android.permission.RECORD_AUDIO" />
|
||||
<uses-permission android:name="android.permission.INTERNET" />
|
||||
<uses-permission android:name="android.permission.ACCESS_NETWORK_STATE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE" />
|
||||
<uses-permission android:name="android.permission.FOREGROUND_SERVICE_CONNECTED_DEVICE" />
|
||||
|
||||
<application
|
||||
android:name=".ClippyAutoApp"
|
||||
android:allowBackup="true"
|
||||
android:icon="@drawable/ic_clippy"
|
||||
android:label="ClippyAuto"
|
||||
android:supportsRtl="true"
|
||||
android:theme="@style/Theme.ClippyAuto">
|
||||
|
||||
<!-- Android Auto CarAppService -->
|
||||
<service
|
||||
android:name=".auto.ClippyCarAppService"
|
||||
android:exported="true">
|
||||
<intent-filter>
|
||||
<action android:name="androidx.car.app.CarAppService" />
|
||||
</intent-filter>
|
||||
</service>
|
||||
|
||||
<meta-data
|
||||
android:name="com.google.android.gms.car.application"
|
||||
android:resource="@xml/automotive_app_desc" />
|
||||
|
||||
<!-- FCM Service -->
|
||||
<service
|
||||
android:name=".push.ClippyFCMService"
|
||||
android:exported="false">
|
||||
<intent-filter>
|
||||
<action android:name="com.google.firebase.MESSAGING_EVENT" />
|
||||
</intent-filter>
|
||||
</service>
|
||||
|
||||
<!-- Phone UI -->
|
||||
<activity
|
||||
android:name=".ui.SetupActivity"
|
||||
android:exported="true">
|
||||
<intent-filter>
|
||||
<action android:name="android.intent.action.MAIN" />
|
||||
<category android:name="android.intent.category.LAUNCHER" />
|
||||
</intent-filter>
|
||||
</activity>
|
||||
</application>
|
||||
</manifest>
|
||||
@@ -0,0 +1,22 @@
|
||||
package farm.hobbs.clippyauto
|
||||
|
||||
import android.app.Application
|
||||
|
||||
class ClippyAutoApp : Application() {
|
||||
|
||||
lateinit var matrixClient: farm.hobbs.clippyauto.matrix.MatrixRestClient
|
||||
private set
|
||||
|
||||
override fun onCreate() {
|
||||
super.onCreate()
|
||||
instance = this
|
||||
matrixClient = farm.hobbs.clippyauto.matrix.MatrixRestClient(
|
||||
homeserverUrl = "https://mess.hobbs.farm"
|
||||
)
|
||||
}
|
||||
|
||||
companion object {
|
||||
lateinit var instance: ClippyAutoApp
|
||||
private set
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,21 @@
|
||||
package farm.hobbs.clippyauto.auto
|
||||
|
||||
import androidx.car.app.CarAppService
|
||||
import androidx.car.app.Session
|
||||
import androidx.car.app.validation.HostValidator
|
||||
|
||||
/**
|
||||
* Entry point for Android Auto.
|
||||
* Creates a ClippySession when the car connects.
|
||||
*/
|
||||
class ClippyCarAppService : CarAppService() {
|
||||
|
||||
override fun createHostValidator(): HostValidator {
|
||||
// Allow all hosts for development; lock down for production
|
||||
return HostValidator.ALLOW_ALL_HOSTS_VALIDATOR
|
||||
}
|
||||
|
||||
override fun onCreateSession(): Session {
|
||||
return ClippySession()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
package farm.hobbs.clippyauto.auto
|
||||
|
||||
import androidx.car.app.CarContext
|
||||
import androidx.car.app.Screen
|
||||
import androidx.car.app.model.*
|
||||
|
||||
/**
|
||||
* The Android Auto screen for ClippyAuto.
|
||||
*
|
||||
* Minimal, driver-safe UI:
|
||||
* - Large "Talk to Clippy" button
|
||||
* - Status indicator (Idle / Listening / Thinking / Speaking)
|
||||
* - Compliant with Android Auto driver distraction guidelines
|
||||
*/
|
||||
class ClippyScreen(carContext: CarContext) : Screen(carContext) {
|
||||
|
||||
private var statusText = "Tap to talk to Clippy"
|
||||
|
||||
override fun onGetTemplate(): Template {
|
||||
val action = Action.Builder()
|
||||
.setTitle("🎙️ Talk")
|
||||
.setOnClickListener {
|
||||
onTalkPressed()
|
||||
}
|
||||
.build()
|
||||
|
||||
val actionStrip = ActionStrip.Builder()
|
||||
.addAction(action)
|
||||
.build()
|
||||
|
||||
val pane = Pane.Builder()
|
||||
.addRow(
|
||||
Row.Builder()
|
||||
.setTitle("📎 Clippy")
|
||||
.addText(statusText)
|
||||
.build()
|
||||
)
|
||||
.addAction(
|
||||
Action.Builder()
|
||||
.setTitle("🎙️ Talk to Clippy")
|
||||
.setOnClickListener { onTalkPressed() }
|
||||
.build()
|
||||
)
|
||||
.build()
|
||||
|
||||
return PaneTemplate.Builder(pane)
|
||||
.setTitle("ClippyAuto")
|
||||
.setHeaderAction(Action.APP_ICON)
|
||||
.setActionStrip(actionStrip)
|
||||
.build()
|
||||
}
|
||||
|
||||
private fun onTalkPressed() {
|
||||
// TODO: Wire to VoiceStateMachine
|
||||
// IDLE → start listening
|
||||
// SPEAKING → interrupt (fade TTS, activate STT)
|
||||
statusText = "Listening..."
|
||||
invalidate() // Refresh the screen
|
||||
}
|
||||
|
||||
fun updateStatus(status: String) {
|
||||
statusText = status
|
||||
invalidate()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package farm.hobbs.clippyauto.auto
|
||||
|
||||
import android.content.Intent
|
||||
import androidx.car.app.Screen
|
||||
import androidx.car.app.Session
|
||||
import androidx.lifecycle.DefaultLifecycleObserver
|
||||
import androidx.lifecycle.LifecycleOwner
|
||||
|
||||
/**
|
||||
* Android Auto session — manages the lifecycle of a Clippy voice interaction.
|
||||
* Sends session_start on create, session_end on destroy.
|
||||
*/
|
||||
class ClippySession : Session(), DefaultLifecycleObserver {
|
||||
|
||||
override fun onCreateScreen(intent: Intent): Screen {
|
||||
lifecycle.addObserver(this)
|
||||
return ClippyScreen(carContext)
|
||||
}
|
||||
|
||||
override fun onStart(owner: LifecycleOwner) {
|
||||
// TODO: Send session_start signal to Matrix #clippy-voice room
|
||||
// TODO: Start Matrix sync (fallback path)
|
||||
// TODO: Subscribe to FCM response bus
|
||||
}
|
||||
|
||||
override fun onStop(owner: LifecycleOwner) {
|
||||
// TODO: Send session_end signal to Matrix
|
||||
// TODO: Clean up voice pipeline
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
package farm.hobbs.clippyauto.connectivity
|
||||
|
||||
import java.util.LinkedList
|
||||
|
||||
/**
|
||||
* Queues outgoing messages during connectivity loss.
|
||||
* Flushes when network returns.
|
||||
*/
|
||||
data class QueuedMessage(
|
||||
val text: String,
|
||||
val timestamp: Long = System.currentTimeMillis()
|
||||
)
|
||||
|
||||
class MessageQueue {
|
||||
private val queue = LinkedList<QueuedMessage>()
|
||||
|
||||
val isEmpty: Boolean get() = queue.isEmpty()
|
||||
val size: Int get() = queue.size
|
||||
|
||||
fun enqueue(text: String) {
|
||||
queue.add(QueuedMessage(text))
|
||||
}
|
||||
|
||||
fun flush(): List<QueuedMessage> {
|
||||
val messages = queue.toList()
|
||||
queue.clear()
|
||||
return messages
|
||||
}
|
||||
|
||||
fun clear() {
|
||||
queue.clear()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
package farm.hobbs.clippyauto.connectivity
|
||||
|
||||
import android.content.Context
|
||||
import android.net.ConnectivityManager
|
||||
import android.net.Network
|
||||
import android.net.NetworkCapabilities
|
||||
import android.net.NetworkRequest
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
|
||||
/**
|
||||
* Monitors network connectivity for graceful offline handling.
|
||||
*
|
||||
* States:
|
||||
* - Online (WiFi or strong cellular)
|
||||
* - Weak (cellular, low signal)
|
||||
* - Offline (no connectivity — tunnel, dead zone)
|
||||
*/
|
||||
enum class NetworkState {
|
||||
ONLINE,
|
||||
WEAK,
|
||||
OFFLINE
|
||||
}
|
||||
|
||||
class NetworkMonitor(context: Context) {
|
||||
|
||||
private val connectivityManager =
|
||||
context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager
|
||||
|
||||
private val _state = MutableStateFlow(NetworkState.ONLINE)
|
||||
val state: StateFlow<NetworkState> = _state.asStateFlow()
|
||||
|
||||
private val callback = object : ConnectivityManager.NetworkCallback() {
|
||||
override fun onAvailable(network: Network) {
|
||||
updateState()
|
||||
}
|
||||
|
||||
override fun onLost(network: Network) {
|
||||
_state.value = NetworkState.OFFLINE
|
||||
}
|
||||
|
||||
override fun onCapabilitiesChanged(
|
||||
network: Network,
|
||||
capabilities: NetworkCapabilities
|
||||
) {
|
||||
updateState()
|
||||
}
|
||||
}
|
||||
|
||||
fun start() {
|
||||
val request = NetworkRequest.Builder()
|
||||
.addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET)
|
||||
.build()
|
||||
connectivityManager.registerNetworkCallback(request, callback)
|
||||
updateState()
|
||||
}
|
||||
|
||||
fun stop() {
|
||||
connectivityManager.unregisterNetworkCallback(callback)
|
||||
}
|
||||
|
||||
private fun updateState() {
|
||||
val network = connectivityManager.activeNetwork
|
||||
val capabilities = network?.let { connectivityManager.getNetworkCapabilities(it) }
|
||||
|
||||
_state.value = when {
|
||||
capabilities == null -> NetworkState.OFFLINE
|
||||
capabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) -> NetworkState.ONLINE
|
||||
capabilities.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR) -> {
|
||||
val downKbps = capabilities.linkDownstreamBandwidthKbps
|
||||
if (downKbps < 1000) NetworkState.WEAK else NetworkState.ONLINE
|
||||
}
|
||||
else -> NetworkState.ONLINE
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,178 @@
|
||||
package farm.hobbs.clippyauto.matrix
|
||||
|
||||
import io.ktor.client.*
|
||||
import io.ktor.client.call.*
|
||||
import io.ktor.client.engine.okhttp.*
|
||||
import io.ktor.client.plugins.contentnegotiation.*
|
||||
import io.ktor.client.request.*
|
||||
import io.ktor.http.*
|
||||
import io.ktor.serialization.kotlinx.json.*
|
||||
import kotlinx.serialization.json.*
|
||||
import java.util.UUID
|
||||
|
||||
/**
|
||||
* Lightweight Matrix REST API client.
|
||||
* No E2EE — designed for private homeserver use.
|
||||
*
|
||||
* Handles: login, message send, long-poll sync (fallback to FCM).
|
||||
*/
|
||||
class MatrixRestClient(
|
||||
private val homeserverUrl: String
|
||||
) {
|
||||
private val httpClient = HttpClient(OkHttp) {
|
||||
install(ContentNegotiation) {
|
||||
json(Json { ignoreUnknownKeys = true })
|
||||
}
|
||||
}
|
||||
|
||||
var accessToken: String? = null
|
||||
private set
|
||||
private var syncToken: String? = null
|
||||
|
||||
/**
|
||||
* Login with username/password, store access token.
|
||||
*/
|
||||
suspend fun login(user: String, password: String): Boolean {
|
||||
return try {
|
||||
val response = httpClient.post("$homeserverUrl/_matrix/client/v3/login") {
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(buildJsonObject {
|
||||
put("type", "m.login.password")
|
||||
put("user", user)
|
||||
put("password", password)
|
||||
putJsonObject("identifier") {
|
||||
put("type", "m.id.user")
|
||||
put("user", user)
|
||||
}
|
||||
})
|
||||
}
|
||||
val body = response.body<JsonObject>()
|
||||
accessToken = body["access_token"]?.jsonPrimitive?.content
|
||||
accessToken != null
|
||||
} catch (e: Exception) {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Login with existing access token (from Keystore).
|
||||
*/
|
||||
fun loginWithToken(token: String) {
|
||||
accessToken = token
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a text message to a room with optional custom fields for session tracking.
|
||||
*/
|
||||
suspend fun sendMessage(
|
||||
roomId: String,
|
||||
body: String,
|
||||
source: String = "voice",
|
||||
sessionId: String? = null
|
||||
): Boolean {
|
||||
val token = accessToken ?: return false
|
||||
val txnId = UUID.randomUUID().toString()
|
||||
return try {
|
||||
httpClient.put("$homeserverUrl/_matrix/client/v3/rooms/$roomId/send/m.room.message/$txnId") {
|
||||
bearerAuth(token)
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(buildJsonObject {
|
||||
put("msgtype", "m.text")
|
||||
put("body", body)
|
||||
put("farm.hobbs.clippy.source", source)
|
||||
sessionId?.let { put("farm.hobbs.clippy.session_id", it) }
|
||||
})
|
||||
}
|
||||
true
|
||||
} catch (e: Exception) {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send a session signal (start, end, resume).
|
||||
*/
|
||||
suspend fun sendSessionSignal(roomId: String, signal: String, sessionId: String): Boolean {
|
||||
val token = accessToken ?: return false
|
||||
val txnId = UUID.randomUUID().toString()
|
||||
val signalBody = when (signal) {
|
||||
"session_start" -> "[ClippyAuto session started — commute mode, voice-first]"
|
||||
"session_resume" -> "[ClippyAuto session resumed]"
|
||||
"session_end" -> "[ClippyAuto session ended]"
|
||||
else -> "[ClippyAuto signal: $signal]"
|
||||
}
|
||||
return try {
|
||||
httpClient.put("$homeserverUrl/_matrix/client/v3/rooms/$roomId/send/m.room.message/$txnId") {
|
||||
bearerAuth(token)
|
||||
contentType(ContentType.Application.Json)
|
||||
setBody(buildJsonObject {
|
||||
put("msgtype", "m.text")
|
||||
put("body", signalBody)
|
||||
put("farm.hobbs.clippy.signal", signal)
|
||||
put("farm.hobbs.clippy.session_id", sessionId)
|
||||
})
|
||||
}
|
||||
true
|
||||
} catch (e: Exception) {
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Long-poll sync — fallback when FCM doesn't deliver.
|
||||
* Returns list of messages from the target room since last sync.
|
||||
*/
|
||||
suspend fun sync(
|
||||
roomId: String,
|
||||
timeoutMs: Int = 30000
|
||||
): List<MatrixMessage> {
|
||||
val token = accessToken ?: return emptyList()
|
||||
return try {
|
||||
val url = buildString {
|
||||
append("$homeserverUrl/_matrix/client/v3/sync?timeout=$timeoutMs")
|
||||
syncToken?.let { append("&since=$it") }
|
||||
}
|
||||
val response = httpClient.get(url) { bearerAuth(token) }
|
||||
val body = response.body<JsonObject>()
|
||||
syncToken = body["next_batch"]?.jsonPrimitive?.content
|
||||
|
||||
val rooms = body["rooms"]?.jsonObject
|
||||
?.get("join")?.jsonObject
|
||||
?.get(roomId)?.jsonObject
|
||||
val timeline = rooms?.get("timeline")?.jsonObject
|
||||
?.get("events")?.jsonArray ?: return emptyList()
|
||||
|
||||
timeline.mapNotNull { event ->
|
||||
val obj = event.jsonObject
|
||||
val type = obj["type"]?.jsonPrimitive?.content
|
||||
if (type != "m.room.message") return@mapNotNull null
|
||||
|
||||
val content = obj["content"]?.jsonObject ?: return@mapNotNull null
|
||||
val sender = obj["sender"]?.jsonPrimitive?.content ?: return@mapNotNull null
|
||||
val msgBody = content["body"]?.jsonPrimitive?.content ?: return@mapNotNull null
|
||||
|
||||
MatrixMessage(
|
||||
sender = sender,
|
||||
body = msgBody,
|
||||
source = content["farm.hobbs.clippy.source"]?.jsonPrimitive?.content,
|
||||
sessionId = content["farm.hobbs.clippy.session_id"]?.jsonPrimitive?.content,
|
||||
signal = content["farm.hobbs.clippy.signal"]?.jsonPrimitive?.content
|
||||
)
|
||||
}
|
||||
} catch (e: Exception) {
|
||||
emptyList()
|
||||
}
|
||||
}
|
||||
|
||||
fun shutdown() {
|
||||
httpClient.close()
|
||||
}
|
||||
}
|
||||
|
||||
data class MatrixMessage(
|
||||
val sender: String,
|
||||
val body: String,
|
||||
val source: String? = null,
|
||||
val sessionId: String? = null,
|
||||
val signal: String? = null
|
||||
)
|
||||
@@ -0,0 +1,76 @@
|
||||
package farm.hobbs.clippyauto.push
|
||||
|
||||
import com.google.firebase.messaging.FirebaseMessagingService
|
||||
import com.google.firebase.messaging.RemoteMessage
|
||||
import java.security.MessageDigest
|
||||
|
||||
/**
|
||||
* Receives FCM data messages from n8n bridge.
|
||||
* Fast path for Clippy responses — bypasses Matrix sync latency.
|
||||
*
|
||||
* Payload format from n8n:
|
||||
* {
|
||||
* "response": "Clippy's text response",
|
||||
* "session_id": "2026-03-05T08:15:00",
|
||||
* "message_hash": "sha256 of response text"
|
||||
* }
|
||||
*/
|
||||
class ClippyFCMService : FirebaseMessagingService() {
|
||||
|
||||
override fun onMessageReceived(message: RemoteMessage) {
|
||||
val responseText = message.data["response"] ?: return
|
||||
val sessionId = message.data["session_id"] ?: ""
|
||||
val hash = sha256(responseText)
|
||||
|
||||
// Dedup: skip if we already got this via Matrix sync
|
||||
if (ResponseDedup.seen(hash)) return
|
||||
ResponseDedup.mark(hash)
|
||||
|
||||
// Notify the voice pipeline to start TTS
|
||||
ClippyResponseBus.emit(responseText, sessionId)
|
||||
}
|
||||
|
||||
override fun onNewToken(token: String) {
|
||||
// TODO: Send updated token to n8n/server for push targeting
|
||||
super.onNewToken(token)
|
||||
}
|
||||
|
||||
private fun sha256(text: String): String {
|
||||
val digest = MessageDigest.getInstance("SHA-256")
|
||||
return digest.digest(text.toByteArray()).joinToString("") { "%02x".format(it) }
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple dedup cache — keeps last N response hashes.
|
||||
* Prevents double-TTS when both FCM and Matrix sync deliver the same message.
|
||||
*/
|
||||
object ResponseDedup {
|
||||
private val cache = LinkedHashSet<String>()
|
||||
private const val MAX_SIZE = 50
|
||||
|
||||
fun seen(hash: String): Boolean = hash in cache
|
||||
|
||||
fun mark(hash: String) {
|
||||
cache.add(hash)
|
||||
if (cache.size > MAX_SIZE) {
|
||||
cache.remove(cache.first())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Simple event bus for FCM → voice pipeline communication.
|
||||
* In production, consider using SharedFlow or LiveData.
|
||||
*/
|
||||
object ClippyResponseBus {
|
||||
private val listeners = mutableListOf<(String, String) -> Unit>()
|
||||
|
||||
fun subscribe(listener: (response: String, sessionId: String) -> Unit) {
|
||||
listeners.add(listener)
|
||||
}
|
||||
|
||||
fun emit(response: String, sessionId: String) {
|
||||
listeners.forEach { it(response, sessionId) }
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
package farm.hobbs.clippyauto.ui
|
||||
|
||||
import android.os.Bundle
|
||||
import androidx.appcompat.app.AppCompatActivity
|
||||
import farm.hobbs.clippyauto.R
|
||||
|
||||
/**
|
||||
* Phone-side activity for initial setup and session monitoring.
|
||||
*
|
||||
* v1 MVP:
|
||||
* - Login to Matrix homeserver (first run)
|
||||
* - Show "session active" indicator during Android Auto use
|
||||
* - Display debrief summary notification after session ends
|
||||
*
|
||||
* v2:
|
||||
* - Full conversation transcript review
|
||||
* - Settings (silence thresholds, TTS voice, etc.)
|
||||
*/
|
||||
class SetupActivity : AppCompatActivity() {
|
||||
|
||||
override fun onCreate(savedInstanceState: Bundle?) {
|
||||
super.onCreate(savedInstanceState)
|
||||
// TODO: Build setup UI
|
||||
// - Matrix homeserver URL (pre-filled: mess.hobbs.farm)
|
||||
// - Username / password fields
|
||||
// - Login button
|
||||
// - Status: connected / disconnected
|
||||
// - "Session active" indicator
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,101 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
import android.content.Context
|
||||
import android.content.Intent
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFocusRequest
|
||||
import android.media.AudioManager
|
||||
import android.os.Bundle
|
||||
import android.speech.RecognitionListener
|
||||
import android.speech.RecognizerIntent
|
||||
import android.speech.SpeechRecognizer
|
||||
|
||||
/**
|
||||
* Manages on-device speech-to-text via Android SpeechRecognizer.
|
||||
*
|
||||
* Key behaviors:
|
||||
* - 2-second silence threshold before sending
|
||||
* - Cancels send if speech resumes within 1s of confirmation chime
|
||||
* - Requests audio focus during listening
|
||||
*/
|
||||
class SpeechManager(
|
||||
private val context: Context,
|
||||
private val onResult: (String) -> Unit,
|
||||
private val onListening: () -> Unit,
|
||||
private val onError: (String) -> Unit
|
||||
) {
|
||||
private var speechRecognizer: SpeechRecognizer? = null
|
||||
private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
|
||||
private val audioFocusRequest = AudioFocusRequest.Builder(
|
||||
AudioManager.AUDIOFOCUS_GAIN_TRANSIENT
|
||||
).setAudioAttributes(
|
||||
AudioAttributes.Builder()
|
||||
.setUsage(AudioAttributes.USAGE_ASSISTANT)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build()
|
||||
).build()
|
||||
|
||||
fun startListening() {
|
||||
audioManager.requestAudioFocus(audioFocusRequest)
|
||||
|
||||
speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context).apply {
|
||||
setRecognitionListener(object : RecognitionListener {
|
||||
override fun onReadyForSpeech(params: Bundle?) { onListening() }
|
||||
override fun onBeginningOfSpeech() {}
|
||||
override fun onRmsChanged(rmsdB: Float) {}
|
||||
override fun onBufferReceived(buffer: ByteArray?) {}
|
||||
override fun onEndOfSpeech() {}
|
||||
|
||||
override fun onResults(results: Bundle?) {
|
||||
val matches = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION)
|
||||
val text = matches?.firstOrNull() ?: return
|
||||
onResult(text)
|
||||
releaseAudioFocus()
|
||||
}
|
||||
|
||||
override fun onPartialResults(partialResults: Bundle?) {
|
||||
// Could use for early display, but not sending partial to Matrix
|
||||
}
|
||||
|
||||
override fun onError(error: Int) {
|
||||
val message = when (error) {
|
||||
SpeechRecognizer.ERROR_NO_MATCH -> "Didn't catch that"
|
||||
SpeechRecognizer.ERROR_NETWORK -> "Network issue"
|
||||
SpeechRecognizer.ERROR_AUDIO -> "Audio error"
|
||||
else -> "Speech error ($error)"
|
||||
}
|
||||
onError(message)
|
||||
releaseAudioFocus()
|
||||
}
|
||||
|
||||
override fun onEvent(eventType: Int, params: Bundle?) {}
|
||||
})
|
||||
}
|
||||
|
||||
val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply {
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM)
|
||||
putExtra(RecognizerIntent.EXTRA_LANGUAGE, "en-US")
|
||||
// 2-second silence = end of utterance
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 2000L)
|
||||
putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, 2000L)
|
||||
}
|
||||
|
||||
speechRecognizer?.startListening(intent)
|
||||
}
|
||||
|
||||
fun stopListening() {
|
||||
speechRecognizer?.stopListening()
|
||||
releaseAudioFocus()
|
||||
}
|
||||
|
||||
fun destroy() {
|
||||
speechRecognizer?.destroy()
|
||||
speechRecognizer = null
|
||||
releaseAudioFocus()
|
||||
}
|
||||
|
||||
private fun releaseAudioFocus() {
|
||||
audioManager.abandonAudioFocusRequest(audioFocusRequest)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,104 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
import android.content.Context
|
||||
import android.media.AudioAttributes
|
||||
import android.media.AudioFocusRequest
|
||||
import android.media.AudioManager
|
||||
import android.speech.tts.TextToSpeech
|
||||
import android.speech.tts.UtteranceProgressListener
|
||||
import java.util.LinkedList
|
||||
import java.util.Locale
|
||||
import java.util.UUID
|
||||
|
||||
/**
|
||||
* Manages text-to-speech output with queuing and audio focus.
|
||||
*
|
||||
* Key behaviors:
|
||||
* - Plays through car audio via USAGE_ASSISTANT
|
||||
* - Ducks music during playback (AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK)
|
||||
* - Queues chunked responses and plays sequentially
|
||||
* - Supports 200ms fade on interruption
|
||||
*/
|
||||
class TTSManager(
|
||||
private val context: Context,
|
||||
private val onSpeakingStart: () -> Unit,
|
||||
private val onSpeakingDone: () -> Unit
|
||||
) {
|
||||
private var tts: TextToSpeech? = null
|
||||
private var isReady = false
|
||||
private val queue = LinkedList<String>()
|
||||
private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager
|
||||
|
||||
private val audioFocusRequest = AudioFocusRequest.Builder(
|
||||
AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK
|
||||
).setAudioAttributes(
|
||||
AudioAttributes.Builder()
|
||||
.setUsage(AudioAttributes.USAGE_ASSISTANT)
|
||||
.setContentType(AudioAttributes.CONTENT_TYPE_SPEECH)
|
||||
.build()
|
||||
).build()
|
||||
|
||||
fun initialize() {
|
||||
tts = TextToSpeech(context) { status ->
|
||||
if (status == TextToSpeech.SUCCESS) {
|
||||
tts?.language = Locale.US
|
||||
tts?.setOnUtteranceProgressListener(object : UtteranceProgressListener() {
|
||||
override fun onStart(utteranceId: String?) {
|
||||
onSpeakingStart()
|
||||
}
|
||||
|
||||
override fun onDone(utteranceId: String?) {
|
||||
if (queue.isNotEmpty()) {
|
||||
speakNext()
|
||||
} else {
|
||||
audioManager.abandonAudioFocusRequest(audioFocusRequest)
|
||||
onSpeakingDone()
|
||||
}
|
||||
}
|
||||
|
||||
@Deprecated("Deprecated in API")
|
||||
override fun onError(utteranceId: String?) {
|
||||
onSpeakingDone()
|
||||
}
|
||||
})
|
||||
isReady = true
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue text for TTS playback. If nothing is currently playing, starts immediately.
|
||||
*/
|
||||
fun speak(text: String) {
|
||||
if (!isReady) return
|
||||
queue.add(text)
|
||||
if (queue.size == 1) {
|
||||
audioManager.requestAudioFocus(audioFocusRequest)
|
||||
speakNext()
|
||||
}
|
||||
}
|
||||
|
||||
private fun speakNext() {
|
||||
val text = queue.peek() ?: return
|
||||
queue.poll()
|
||||
val utteranceId = UUID.randomUUID().toString()
|
||||
tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, utteranceId)
|
||||
}
|
||||
|
||||
/**
|
||||
* Interrupt current speech (for when Jeremy starts talking).
|
||||
* Clears the queue so unplayed chunks are dropped.
|
||||
*/
|
||||
fun interrupt() {
|
||||
queue.clear()
|
||||
tts?.stop()
|
||||
audioManager.abandonAudioFocusRequest(audioFocusRequest)
|
||||
}
|
||||
|
||||
fun shutdown() {
|
||||
tts?.stop()
|
||||
tts?.shutdown()
|
||||
tts = null
|
||||
queue.clear()
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,29 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
/**
|
||||
* Voice interaction state machine.
|
||||
*
|
||||
* IDLE ──(tap button)──► LISTENING
|
||||
* LISTENING ──(2s silence + chime)──► PROCESSING
|
||||
* PROCESSING ──(sent to Matrix)──► RECEIVING
|
||||
* RECEIVING ──(response arrived)──► SPEAKING
|
||||
* SPEAKING ──(TTS complete)──► IDLE
|
||||
*
|
||||
* Interruption: SPEAKING ──(voice detected)──► LISTENING (200ms TTS fade)
|
||||
*/
|
||||
enum class VoiceState {
|
||||
/** Ready, showing "Tap to talk" */
|
||||
IDLE,
|
||||
|
||||
/** Mic active, STT running */
|
||||
LISTENING,
|
||||
|
||||
/** Utterance sent to Matrix, waiting for Clippy */
|
||||
PROCESSING,
|
||||
|
||||
/** Response received from Clippy, preparing TTS */
|
||||
RECEIVING,
|
||||
|
||||
/** TTS playing through car speakers */
|
||||
SPEAKING
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package farm.hobbs.clippyauto.voice
|
||||
|
||||
import kotlinx.coroutines.flow.MutableStateFlow
|
||||
import kotlinx.coroutines.flow.StateFlow
|
||||
import kotlinx.coroutines.flow.asStateFlow
|
||||
|
||||
/**
|
||||
* Manages voice interaction state transitions.
|
||||
* Enforces valid transitions and notifies observers.
|
||||
*/
|
||||
class VoiceStateMachine {
|
||||
|
||||
private val _state = MutableStateFlow(VoiceState.IDLE)
|
||||
val state: StateFlow<VoiceState> = _state.asStateFlow()
|
||||
|
||||
fun transition(newState: VoiceState): Boolean {
|
||||
val current = _state.value
|
||||
if (!isValidTransition(current, newState)) return false
|
||||
_state.value = newState
|
||||
return true
|
||||
}
|
||||
|
||||
private fun isValidTransition(from: VoiceState, to: VoiceState): Boolean {
|
||||
return when (from) {
|
||||
VoiceState.IDLE -> to == VoiceState.LISTENING
|
||||
VoiceState.LISTENING -> to in setOf(VoiceState.PROCESSING, VoiceState.IDLE)
|
||||
VoiceState.PROCESSING -> to in setOf(VoiceState.RECEIVING, VoiceState.IDLE)
|
||||
VoiceState.RECEIVING -> to in setOf(VoiceState.SPEAKING, VoiceState.IDLE)
|
||||
VoiceState.SPEAKING -> to in setOf(VoiceState.IDLE, VoiceState.LISTENING) // LISTENING = interruption
|
||||
}
|
||||
}
|
||||
|
||||
fun reset() {
|
||||
_state.value = VoiceState.IDLE
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<string name="app_name">ClippyAuto</string>
|
||||
</resources>
|
||||
@@ -0,0 +1,8 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<resources>
|
||||
<style name="Theme.ClippyAuto" parent="Theme.AppCompat.Light.DarkActionBar">
|
||||
<item name="colorPrimary">#1976D2</item>
|
||||
<item name="colorPrimaryDark">#1565C0</item>
|
||||
<item name="colorAccent">#FF9800</item>
|
||||
</style>
|
||||
</resources>
|
||||
@@ -0,0 +1,4 @@
|
||||
<?xml version="1.0" encoding="utf-8"?>
|
||||
<automotiveApp>
|
||||
<uses name="template" />
|
||||
</automotiveApp>
|
||||
Reference in New Issue
Block a user