diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..aca6ecc --- /dev/null +++ b/.gitignore @@ -0,0 +1,13 @@ +*.iml +.gradle +/local.properties +/.idea +.DS_Store +/build +/captures +.externalNativeBuild +.cxx +local.properties +google-services.json +*.apk +*.aab diff --git a/README.md b/README.md index 855508d..0f17124 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,104 @@ -# clippy-auto +# ClippyAuto πŸ“ŽπŸš— -Voice-first Android Auto app for Clippy via Matrix \ No newline at end of file +Voice-first Android Auto app for talking to Clippy via Matrix. + +## Overview + +ClippyAuto gives Jeremy a hands-free voice conversation with Clippy (AI Chief of Staff) during his commute. No typing, no "Hey Google," no Element β€” just tap and talk. + +**How it works:** You talk β†’ on-device STT β†’ Matrix message to Clippy β†’ Claude responds β†’ FCM push β†’ car speakers read it back. Target: under 5 seconds to first audio. + +## Prerequisites + +- Android Studio Ladybug (2024.2+) +- JDK 17 +- Android device with Android Auto support (or Desktop Head Unit for testing) +- Firebase project with FCM enabled +- Access to `mess.hobbs.farm` Matrix homeserver + +## Setup + +### 1. Clone & Open +```bash +git clone https://git.hobbs.farm/spirit/clippy-auto.git +cd clippy-auto +# Open in Android Studio +``` + +### 2. Firebase +1. Create a Firebase project at https://console.firebase.google.com +2. Add an Android app with package name `farm.hobbs.clippyauto` +3. Download `google-services.json` β†’ place in `app/` +4. Enable Cloud Messaging (v1 API) + +### 3. Matrix Homeserver +- `mess.hobbs.farm` must be reachable externally (HTTPS via Traefik) βœ… +- Jeremy's account needs access to `#clippy-voice:hobbs.farm` room + +### 4. n8n FCM Bridge +- Import `n8n/clippy-fcm-bridge.json` workflow into n8n +- Register Matrix pusher on Jeremy's account pointing to n8n webhook +- Configure FCM server key in n8n credentials + +### 5. Build & Run +```bash +./gradlew installDebug +``` + +### Testing with Desktop Head Unit (DHU) +```bash +$ANDROID_HOME/extras/google/auto/desktop-head-unit +``` + +## Project Structure + +``` +app/src/main/java/farm/hobbs/clippyauto/ +β”œβ”€β”€ ClippyAutoApp.kt # Application entry, DI +β”œβ”€β”€ auto/ +β”‚ β”œβ”€β”€ ClippyCarAppService.kt # Android Auto entry point +β”‚ β”œβ”€β”€ ClippySession.kt # Auto session lifecycle +β”‚ └── ClippyScreen.kt # Auto UI (talk button + status) +β”œβ”€β”€ voice/ +β”‚ β”œβ”€β”€ VoiceState.kt # State enum +β”‚ β”œβ”€β”€ VoiceStateMachine.kt # State transitions +β”‚ β”œβ”€β”€ SpeechManager.kt # STT (SpeechRecognizer) +β”‚ └── TTSManager.kt # TTS with queue + audio focus +β”œβ”€β”€ matrix/ +β”‚ └── MatrixRestClient.kt # Lightweight Matrix REST API client +β”œβ”€β”€ push/ +β”‚ └── ClippyFCMService.kt # FCM receiver + dedup +β”œβ”€β”€ connectivity/ +β”‚ β”œβ”€β”€ NetworkMonitor.kt # Online/Weak/Offline detection +β”‚ └── MessageQueue.kt # Offline message queuing +└── ui/ + └── SetupActivity.kt # Phone setup + session indicator +``` + +## Architecture + +See [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md) for the full spec. + +## Status + +🚧 **v1 MVP in progress** + +- [x] Project scaffold +- [x] Android Auto CarAppService + Screen +- [x] Voice state machine +- [x] STT manager (SpeechRecognizer) +- [x] TTS manager (queue + audio focus + interruption) +- [x] Matrix REST client (login, send, sync) +- [x] FCM push receiver + dedup +- [x] Network monitor + message queue +- [ ] Wire voice pipeline end-to-end +- [ ] Phone setup UI (login flow) +- [ ] n8n FCM bridge workflow +- [ ] Confirmation chime + thinking tone audio assets +- [ ] Session lifecycle signals +- [ ] Voice command recognition (go quiet, wrap up, etc.) +- [ ] Real-car testing + +## License + +Private β€” hobbs.farm internal use. diff --git a/app/build.gradle.kts b/app/build.gradle.kts new file mode 100644 index 0000000..49d8575 --- /dev/null +++ b/app/build.gradle.kts @@ -0,0 +1,66 @@ +plugins { + id("com.android.application") + id("org.jetbrains.kotlin.android") + id("com.google.gms.google-services") +} + +android { + namespace = "farm.hobbs.clippyauto" + compileSdk = 35 + + defaultConfig { + applicationId = "farm.hobbs.clippyauto" + minSdk = 28 + targetSdk = 35 + versionCode = 1 + versionName = "0.1.0-mvp" + } + + buildTypes { + release { + isMinifyEnabled = false + proguardFiles(getDefaultProguardFile("proguard-android-optimize.txt"), "proguard-rules.pro") + } + } + + compileOptions { + sourceCompatibility = JavaVersion.VERSION_17 + targetCompatibility = JavaVersion.VERSION_17 + } + + kotlinOptions { + jvmTarget = "17" + } +} + +dependencies { + // Android Auto + implementation("androidx.car.app:app:1.7.0") + implementation("androidx.car.app:app-projected:1.7.0") + + // Firebase Cloud Messaging + implementation(platform("com.google.firebase:firebase-bom:33.9.0")) + implementation("com.google.firebase:firebase-messaging-ktx") + + // HTTP client for Matrix REST API + implementation("io.ktor:ktor-client-okhttp:3.0.3") + implementation("io.ktor:ktor-client-content-negotiation:3.0.3") + implementation("io.ktor:ktor-serialization-kotlinx-json:3.0.3") + + // Coroutines + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-android:1.9.0") + + // Lifecycle + implementation("androidx.lifecycle:lifecycle-runtime-ktx:2.8.7") + implementation("androidx.lifecycle:lifecycle-service:2.8.7") + + // DataStore (settings) + implementation("androidx.datastore:datastore-preferences:1.1.3") + + // Security (Keystore wrapper) + implementation("androidx.security:security-crypto:1.1.0-alpha06") + + // Core + implementation("androidx.core:core-ktx:1.15.0") + implementation("androidx.appcompat:appcompat:1.7.0") +} diff --git a/app/src/main/AndroidManifest.xml b/app/src/main/AndroidManifest.xml new file mode 100644 index 0000000..6afd3d8 --- /dev/null +++ b/app/src/main/AndroidManifest.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/app/src/main/java/farm/hobbs/clippyauto/ClippyAutoApp.kt b/app/src/main/java/farm/hobbs/clippyauto/ClippyAutoApp.kt new file mode 100644 index 0000000..a8f77f9 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/ClippyAutoApp.kt @@ -0,0 +1,22 @@ +package farm.hobbs.clippyauto + +import android.app.Application + +class ClippyAutoApp : Application() { + + lateinit var matrixClient: farm.hobbs.clippyauto.matrix.MatrixRestClient + private set + + override fun onCreate() { + super.onCreate() + instance = this + matrixClient = farm.hobbs.clippyauto.matrix.MatrixRestClient( + homeserverUrl = "https://mess.hobbs.farm" + ) + } + + companion object { + lateinit var instance: ClippyAutoApp + private set + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/auto/ClippyCarAppService.kt b/app/src/main/java/farm/hobbs/clippyauto/auto/ClippyCarAppService.kt new file mode 100644 index 0000000..a498974 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/auto/ClippyCarAppService.kt @@ -0,0 +1,21 @@ +package farm.hobbs.clippyauto.auto + +import androidx.car.app.CarAppService +import androidx.car.app.Session +import androidx.car.app.validation.HostValidator + +/** + * Entry point for Android Auto. + * Creates a ClippySession when the car connects. + */ +class ClippyCarAppService : CarAppService() { + + override fun createHostValidator(): HostValidator { + // Allow all hosts for development; lock down for production + return HostValidator.ALLOW_ALL_HOSTS_VALIDATOR + } + + override fun onCreateSession(): Session { + return ClippySession() + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/auto/ClippyScreen.kt b/app/src/main/java/farm/hobbs/clippyauto/auto/ClippyScreen.kt new file mode 100644 index 0000000..ad6ef32 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/auto/ClippyScreen.kt @@ -0,0 +1,65 @@ +package farm.hobbs.clippyauto.auto + +import androidx.car.app.CarContext +import androidx.car.app.Screen +import androidx.car.app.model.* + +/** + * The Android Auto screen for ClippyAuto. + * + * Minimal, driver-safe UI: + * - Large "Talk to Clippy" button + * - Status indicator (Idle / Listening / Thinking / Speaking) + * - Compliant with Android Auto driver distraction guidelines + */ +class ClippyScreen(carContext: CarContext) : Screen(carContext) { + + private var statusText = "Tap to talk to Clippy" + + override fun onGetTemplate(): Template { + val action = Action.Builder() + .setTitle("πŸŽ™οΈ Talk") + .setOnClickListener { + onTalkPressed() + } + .build() + + val actionStrip = ActionStrip.Builder() + .addAction(action) + .build() + + val pane = Pane.Builder() + .addRow( + Row.Builder() + .setTitle("πŸ“Ž Clippy") + .addText(statusText) + .build() + ) + .addAction( + Action.Builder() + .setTitle("πŸŽ™οΈ Talk to Clippy") + .setOnClickListener { onTalkPressed() } + .build() + ) + .build() + + return PaneTemplate.Builder(pane) + .setTitle("ClippyAuto") + .setHeaderAction(Action.APP_ICON) + .setActionStrip(actionStrip) + .build() + } + + private fun onTalkPressed() { + // TODO: Wire to VoiceStateMachine + // IDLE β†’ start listening + // SPEAKING β†’ interrupt (fade TTS, activate STT) + statusText = "Listening..." + invalidate() // Refresh the screen + } + + fun updateStatus(status: String) { + statusText = status + invalidate() + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/auto/ClippySession.kt b/app/src/main/java/farm/hobbs/clippyauto/auto/ClippySession.kt new file mode 100644 index 0000000..5cf6328 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/auto/ClippySession.kt @@ -0,0 +1,30 @@ +package farm.hobbs.clippyauto.auto + +import android.content.Intent +import androidx.car.app.Screen +import androidx.car.app.Session +import androidx.lifecycle.DefaultLifecycleObserver +import androidx.lifecycle.LifecycleOwner + +/** + * Android Auto session β€” manages the lifecycle of a Clippy voice interaction. + * Sends session_start on create, session_end on destroy. + */ +class ClippySession : Session(), DefaultLifecycleObserver { + + override fun onCreateScreen(intent: Intent): Screen { + lifecycle.addObserver(this) + return ClippyScreen(carContext) + } + + override fun onStart(owner: LifecycleOwner) { + // TODO: Send session_start signal to Matrix #clippy-voice room + // TODO: Start Matrix sync (fallback path) + // TODO: Subscribe to FCM response bus + } + + override fun onStop(owner: LifecycleOwner) { + // TODO: Send session_end signal to Matrix + // TODO: Clean up voice pipeline + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/connectivity/MessageQueue.kt b/app/src/main/java/farm/hobbs/clippyauto/connectivity/MessageQueue.kt new file mode 100644 index 0000000..5a4014e --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/connectivity/MessageQueue.kt @@ -0,0 +1,33 @@ +package farm.hobbs.clippyauto.connectivity + +import java.util.LinkedList + +/** + * Queues outgoing messages during connectivity loss. + * Flushes when network returns. + */ +data class QueuedMessage( + val text: String, + val timestamp: Long = System.currentTimeMillis() +) + +class MessageQueue { + private val queue = LinkedList() + + val isEmpty: Boolean get() = queue.isEmpty() + val size: Int get() = queue.size + + fun enqueue(text: String) { + queue.add(QueuedMessage(text)) + } + + fun flush(): List { + val messages = queue.toList() + queue.clear() + return messages + } + + fun clear() { + queue.clear() + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/connectivity/NetworkMonitor.kt b/app/src/main/java/farm/hobbs/clippyauto/connectivity/NetworkMonitor.kt new file mode 100644 index 0000000..0d6159a --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/connectivity/NetworkMonitor.kt @@ -0,0 +1,77 @@ +package farm.hobbs.clippyauto.connectivity + +import android.content.Context +import android.net.ConnectivityManager +import android.net.Network +import android.net.NetworkCapabilities +import android.net.NetworkRequest +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** + * Monitors network connectivity for graceful offline handling. + * + * States: + * - Online (WiFi or strong cellular) + * - Weak (cellular, low signal) + * - Offline (no connectivity β€” tunnel, dead zone) + */ +enum class NetworkState { + ONLINE, + WEAK, + OFFLINE +} + +class NetworkMonitor(context: Context) { + + private val connectivityManager = + context.getSystemService(Context.CONNECTIVITY_SERVICE) as ConnectivityManager + + private val _state = MutableStateFlow(NetworkState.ONLINE) + val state: StateFlow = _state.asStateFlow() + + private val callback = object : ConnectivityManager.NetworkCallback() { + override fun onAvailable(network: Network) { + updateState() + } + + override fun onLost(network: Network) { + _state.value = NetworkState.OFFLINE + } + + override fun onCapabilitiesChanged( + network: Network, + capabilities: NetworkCapabilities + ) { + updateState() + } + } + + fun start() { + val request = NetworkRequest.Builder() + .addCapability(NetworkCapabilities.NET_CAPABILITY_INTERNET) + .build() + connectivityManager.registerNetworkCallback(request, callback) + updateState() + } + + fun stop() { + connectivityManager.unregisterNetworkCallback(callback) + } + + private fun updateState() { + val network = connectivityManager.activeNetwork + val capabilities = network?.let { connectivityManager.getNetworkCapabilities(it) } + + _state.value = when { + capabilities == null -> NetworkState.OFFLINE + capabilities.hasTransport(NetworkCapabilities.TRANSPORT_WIFI) -> NetworkState.ONLINE + capabilities.hasTransport(NetworkCapabilities.TRANSPORT_CELLULAR) -> { + val downKbps = capabilities.linkDownstreamBandwidthKbps + if (downKbps < 1000) NetworkState.WEAK else NetworkState.ONLINE + } + else -> NetworkState.ONLINE + } + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/matrix/MatrixRestClient.kt b/app/src/main/java/farm/hobbs/clippyauto/matrix/MatrixRestClient.kt new file mode 100644 index 0000000..fe9d179 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/matrix/MatrixRestClient.kt @@ -0,0 +1,178 @@ +package farm.hobbs.clippyauto.matrix + +import io.ktor.client.* +import io.ktor.client.call.* +import io.ktor.client.engine.okhttp.* +import io.ktor.client.plugins.contentnegotiation.* +import io.ktor.client.request.* +import io.ktor.http.* +import io.ktor.serialization.kotlinx.json.* +import kotlinx.serialization.json.* +import java.util.UUID + +/** + * Lightweight Matrix REST API client. + * No E2EE β€” designed for private homeserver use. + * + * Handles: login, message send, long-poll sync (fallback to FCM). + */ +class MatrixRestClient( + private val homeserverUrl: String +) { + private val httpClient = HttpClient(OkHttp) { + install(ContentNegotiation) { + json(Json { ignoreUnknownKeys = true }) + } + } + + var accessToken: String? = null + private set + private var syncToken: String? = null + + /** + * Login with username/password, store access token. + */ + suspend fun login(user: String, password: String): Boolean { + return try { + val response = httpClient.post("$homeserverUrl/_matrix/client/v3/login") { + contentType(ContentType.Application.Json) + setBody(buildJsonObject { + put("type", "m.login.password") + put("user", user) + put("password", password) + putJsonObject("identifier") { + put("type", "m.id.user") + put("user", user) + } + }) + } + val body = response.body() + accessToken = body["access_token"]?.jsonPrimitive?.content + accessToken != null + } catch (e: Exception) { + false + } + } + + /** + * Login with existing access token (from Keystore). + */ + fun loginWithToken(token: String) { + accessToken = token + } + + /** + * Send a text message to a room with optional custom fields for session tracking. + */ + suspend fun sendMessage( + roomId: String, + body: String, + source: String = "voice", + sessionId: String? = null + ): Boolean { + val token = accessToken ?: return false + val txnId = UUID.randomUUID().toString() + return try { + httpClient.put("$homeserverUrl/_matrix/client/v3/rooms/$roomId/send/m.room.message/$txnId") { + bearerAuth(token) + contentType(ContentType.Application.Json) + setBody(buildJsonObject { + put("msgtype", "m.text") + put("body", body) + put("farm.hobbs.clippy.source", source) + sessionId?.let { put("farm.hobbs.clippy.session_id", it) } + }) + } + true + } catch (e: Exception) { + false + } + } + + /** + * Send a session signal (start, end, resume). + */ + suspend fun sendSessionSignal(roomId: String, signal: String, sessionId: String): Boolean { + val token = accessToken ?: return false + val txnId = UUID.randomUUID().toString() + val signalBody = when (signal) { + "session_start" -> "[ClippyAuto session started β€” commute mode, voice-first]" + "session_resume" -> "[ClippyAuto session resumed]" + "session_end" -> "[ClippyAuto session ended]" + else -> "[ClippyAuto signal: $signal]" + } + return try { + httpClient.put("$homeserverUrl/_matrix/client/v3/rooms/$roomId/send/m.room.message/$txnId") { + bearerAuth(token) + contentType(ContentType.Application.Json) + setBody(buildJsonObject { + put("msgtype", "m.text") + put("body", signalBody) + put("farm.hobbs.clippy.signal", signal) + put("farm.hobbs.clippy.session_id", sessionId) + }) + } + true + } catch (e: Exception) { + false + } + } + + /** + * Long-poll sync β€” fallback when FCM doesn't deliver. + * Returns list of messages from the target room since last sync. + */ + suspend fun sync( + roomId: String, + timeoutMs: Int = 30000 + ): List { + val token = accessToken ?: return emptyList() + return try { + val url = buildString { + append("$homeserverUrl/_matrix/client/v3/sync?timeout=$timeoutMs") + syncToken?.let { append("&since=$it") } + } + val response = httpClient.get(url) { bearerAuth(token) } + val body = response.body() + syncToken = body["next_batch"]?.jsonPrimitive?.content + + val rooms = body["rooms"]?.jsonObject + ?.get("join")?.jsonObject + ?.get(roomId)?.jsonObject + val timeline = rooms?.get("timeline")?.jsonObject + ?.get("events")?.jsonArray ?: return emptyList() + + timeline.mapNotNull { event -> + val obj = event.jsonObject + val type = obj["type"]?.jsonPrimitive?.content + if (type != "m.room.message") return@mapNotNull null + + val content = obj["content"]?.jsonObject ?: return@mapNotNull null + val sender = obj["sender"]?.jsonPrimitive?.content ?: return@mapNotNull null + val msgBody = content["body"]?.jsonPrimitive?.content ?: return@mapNotNull null + + MatrixMessage( + sender = sender, + body = msgBody, + source = content["farm.hobbs.clippy.source"]?.jsonPrimitive?.content, + sessionId = content["farm.hobbs.clippy.session_id"]?.jsonPrimitive?.content, + signal = content["farm.hobbs.clippy.signal"]?.jsonPrimitive?.content + ) + } + } catch (e: Exception) { + emptyList() + } + } + + fun shutdown() { + httpClient.close() + } +} + +data class MatrixMessage( + val sender: String, + val body: String, + val source: String? = null, + val sessionId: String? = null, + val signal: String? = null +) diff --git a/app/src/main/java/farm/hobbs/clippyauto/push/ClippyFCMService.kt b/app/src/main/java/farm/hobbs/clippyauto/push/ClippyFCMService.kt new file mode 100644 index 0000000..d00bb0b --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/push/ClippyFCMService.kt @@ -0,0 +1,76 @@ +package farm.hobbs.clippyauto.push + +import com.google.firebase.messaging.FirebaseMessagingService +import com.google.firebase.messaging.RemoteMessage +import java.security.MessageDigest + +/** + * Receives FCM data messages from n8n bridge. + * Fast path for Clippy responses β€” bypasses Matrix sync latency. + * + * Payload format from n8n: + * { + * "response": "Clippy's text response", + * "session_id": "2026-03-05T08:15:00", + * "message_hash": "sha256 of response text" + * } + */ +class ClippyFCMService : FirebaseMessagingService() { + + override fun onMessageReceived(message: RemoteMessage) { + val responseText = message.data["response"] ?: return + val sessionId = message.data["session_id"] ?: "" + val hash = sha256(responseText) + + // Dedup: skip if we already got this via Matrix sync + if (ResponseDedup.seen(hash)) return + ResponseDedup.mark(hash) + + // Notify the voice pipeline to start TTS + ClippyResponseBus.emit(responseText, sessionId) + } + + override fun onNewToken(token: String) { + // TODO: Send updated token to n8n/server for push targeting + super.onNewToken(token) + } + + private fun sha256(text: String): String { + val digest = MessageDigest.getInstance("SHA-256") + return digest.digest(text.toByteArray()).joinToString("") { "%02x".format(it) } + } +} + +/** + * Simple dedup cache β€” keeps last N response hashes. + * Prevents double-TTS when both FCM and Matrix sync deliver the same message. + */ +object ResponseDedup { + private val cache = LinkedHashSet() + private const val MAX_SIZE = 50 + + fun seen(hash: String): Boolean = hash in cache + + fun mark(hash: String) { + cache.add(hash) + if (cache.size > MAX_SIZE) { + cache.remove(cache.first()) + } + } +} + +/** + * Simple event bus for FCM β†’ voice pipeline communication. + * In production, consider using SharedFlow or LiveData. + */ +object ClippyResponseBus { + private val listeners = mutableListOf<(String, String) -> Unit>() + + fun subscribe(listener: (response: String, sessionId: String) -> Unit) { + listeners.add(listener) + } + + fun emit(response: String, sessionId: String) { + listeners.forEach { it(response, sessionId) } + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/ui/SetupActivity.kt b/app/src/main/java/farm/hobbs/clippyauto/ui/SetupActivity.kt new file mode 100644 index 0000000..4d40282 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/ui/SetupActivity.kt @@ -0,0 +1,30 @@ +package farm.hobbs.clippyauto.ui + +import android.os.Bundle +import androidx.appcompat.app.AppCompatActivity +import farm.hobbs.clippyauto.R + +/** + * Phone-side activity for initial setup and session monitoring. + * + * v1 MVP: + * - Login to Matrix homeserver (first run) + * - Show "session active" indicator during Android Auto use + * - Display debrief summary notification after session ends + * + * v2: + * - Full conversation transcript review + * - Settings (silence thresholds, TTS voice, etc.) + */ +class SetupActivity : AppCompatActivity() { + + override fun onCreate(savedInstanceState: Bundle?) { + super.onCreate(savedInstanceState) + // TODO: Build setup UI + // - Matrix homeserver URL (pre-filled: mess.hobbs.farm) + // - Username / password fields + // - Login button + // - Status: connected / disconnected + // - "Session active" indicator + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/voice/SpeechManager.kt b/app/src/main/java/farm/hobbs/clippyauto/voice/SpeechManager.kt new file mode 100644 index 0000000..29b1096 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/voice/SpeechManager.kt @@ -0,0 +1,101 @@ +package farm.hobbs.clippyauto.voice + +import android.content.Context +import android.content.Intent +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.os.Bundle +import android.speech.RecognitionListener +import android.speech.RecognizerIntent +import android.speech.SpeechRecognizer + +/** + * Manages on-device speech-to-text via Android SpeechRecognizer. + * + * Key behaviors: + * - 2-second silence threshold before sending + * - Cancels send if speech resumes within 1s of confirmation chime + * - Requests audio focus during listening + */ +class SpeechManager( + private val context: Context, + private val onResult: (String) -> Unit, + private val onListening: () -> Unit, + private val onError: (String) -> Unit +) { + private var speechRecognizer: SpeechRecognizer? = null + private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + private val audioFocusRequest = AudioFocusRequest.Builder( + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT + ).setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANT) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ).build() + + fun startListening() { + audioManager.requestAudioFocus(audioFocusRequest) + + speechRecognizer = SpeechRecognizer.createSpeechRecognizer(context).apply { + setRecognitionListener(object : RecognitionListener { + override fun onReadyForSpeech(params: Bundle?) { onListening() } + override fun onBeginningOfSpeech() {} + override fun onRmsChanged(rmsdB: Float) {} + override fun onBufferReceived(buffer: ByteArray?) {} + override fun onEndOfSpeech() {} + + override fun onResults(results: Bundle?) { + val matches = results?.getStringArrayList(SpeechRecognizer.RESULTS_RECOGNITION) + val text = matches?.firstOrNull() ?: return + onResult(text) + releaseAudioFocus() + } + + override fun onPartialResults(partialResults: Bundle?) { + // Could use for early display, but not sending partial to Matrix + } + + override fun onError(error: Int) { + val message = when (error) { + SpeechRecognizer.ERROR_NO_MATCH -> "Didn't catch that" + SpeechRecognizer.ERROR_NETWORK -> "Network issue" + SpeechRecognizer.ERROR_AUDIO -> "Audio error" + else -> "Speech error ($error)" + } + onError(message) + releaseAudioFocus() + } + + override fun onEvent(eventType: Int, params: Bundle?) {} + }) + } + + val intent = Intent(RecognizerIntent.ACTION_RECOGNIZE_SPEECH).apply { + putExtra(RecognizerIntent.EXTRA_LANGUAGE_MODEL, RecognizerIntent.LANGUAGE_MODEL_FREE_FORM) + putExtra(RecognizerIntent.EXTRA_LANGUAGE, "en-US") + // 2-second silence = end of utterance + putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_COMPLETE_SILENCE_LENGTH_MILLIS, 2000L) + putExtra(RecognizerIntent.EXTRA_SPEECH_INPUT_POSSIBLY_COMPLETE_SILENCE_LENGTH_MILLIS, 2000L) + } + + speechRecognizer?.startListening(intent) + } + + fun stopListening() { + speechRecognizer?.stopListening() + releaseAudioFocus() + } + + fun destroy() { + speechRecognizer?.destroy() + speechRecognizer = null + releaseAudioFocus() + } + + private fun releaseAudioFocus() { + audioManager.abandonAudioFocusRequest(audioFocusRequest) + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/voice/TTSManager.kt b/app/src/main/java/farm/hobbs/clippyauto/voice/TTSManager.kt new file mode 100644 index 0000000..3a42017 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/voice/TTSManager.kt @@ -0,0 +1,104 @@ +package farm.hobbs.clippyauto.voice + +import android.content.Context +import android.media.AudioAttributes +import android.media.AudioFocusRequest +import android.media.AudioManager +import android.speech.tts.TextToSpeech +import android.speech.tts.UtteranceProgressListener +import java.util.LinkedList +import java.util.Locale +import java.util.UUID + +/** + * Manages text-to-speech output with queuing and audio focus. + * + * Key behaviors: + * - Plays through car audio via USAGE_ASSISTANT + * - Ducks music during playback (AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK) + * - Queues chunked responses and plays sequentially + * - Supports 200ms fade on interruption + */ +class TTSManager( + private val context: Context, + private val onSpeakingStart: () -> Unit, + private val onSpeakingDone: () -> Unit +) { + private var tts: TextToSpeech? = null + private var isReady = false + private val queue = LinkedList() + private val audioManager = context.getSystemService(Context.AUDIO_SERVICE) as AudioManager + + private val audioFocusRequest = AudioFocusRequest.Builder( + AudioManager.AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK + ).setAudioAttributes( + AudioAttributes.Builder() + .setUsage(AudioAttributes.USAGE_ASSISTANT) + .setContentType(AudioAttributes.CONTENT_TYPE_SPEECH) + .build() + ).build() + + fun initialize() { + tts = TextToSpeech(context) { status -> + if (status == TextToSpeech.SUCCESS) { + tts?.language = Locale.US + tts?.setOnUtteranceProgressListener(object : UtteranceProgressListener() { + override fun onStart(utteranceId: String?) { + onSpeakingStart() + } + + override fun onDone(utteranceId: String?) { + if (queue.isNotEmpty()) { + speakNext() + } else { + audioManager.abandonAudioFocusRequest(audioFocusRequest) + onSpeakingDone() + } + } + + @Deprecated("Deprecated in API") + override fun onError(utteranceId: String?) { + onSpeakingDone() + } + }) + isReady = true + } + } + } + + /** + * Queue text for TTS playback. If nothing is currently playing, starts immediately. + */ + fun speak(text: String) { + if (!isReady) return + queue.add(text) + if (queue.size == 1) { + audioManager.requestAudioFocus(audioFocusRequest) + speakNext() + } + } + + private fun speakNext() { + val text = queue.peek() ?: return + queue.poll() + val utteranceId = UUID.randomUUID().toString() + tts?.speak(text, TextToSpeech.QUEUE_FLUSH, null, utteranceId) + } + + /** + * Interrupt current speech (for when Jeremy starts talking). + * Clears the queue so unplayed chunks are dropped. + */ + fun interrupt() { + queue.clear() + tts?.stop() + audioManager.abandonAudioFocusRequest(audioFocusRequest) + } + + fun shutdown() { + tts?.stop() + tts?.shutdown() + tts = null + queue.clear() + } +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/voice/VoiceState.kt b/app/src/main/java/farm/hobbs/clippyauto/voice/VoiceState.kt new file mode 100644 index 0000000..fc00a71 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/voice/VoiceState.kt @@ -0,0 +1,29 @@ +package farm.hobbs.clippyauto.voice + +/** + * Voice interaction state machine. + * + * IDLE ──(tap button)──► LISTENING + * LISTENING ──(2s silence + chime)──► PROCESSING + * PROCESSING ──(sent to Matrix)──► RECEIVING + * RECEIVING ──(response arrived)──► SPEAKING + * SPEAKING ──(TTS complete)──► IDLE + * + * Interruption: SPEAKING ──(voice detected)──► LISTENING (200ms TTS fade) + */ +enum class VoiceState { + /** Ready, showing "Tap to talk" */ + IDLE, + + /** Mic active, STT running */ + LISTENING, + + /** Utterance sent to Matrix, waiting for Clippy */ + PROCESSING, + + /** Response received from Clippy, preparing TTS */ + RECEIVING, + + /** TTS playing through car speakers */ + SPEAKING +} diff --git a/app/src/main/java/farm/hobbs/clippyauto/voice/VoiceStateMachine.kt b/app/src/main/java/farm/hobbs/clippyauto/voice/VoiceStateMachine.kt new file mode 100644 index 0000000..f184c95 --- /dev/null +++ b/app/src/main/java/farm/hobbs/clippyauto/voice/VoiceStateMachine.kt @@ -0,0 +1,36 @@ +package farm.hobbs.clippyauto.voice + +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow + +/** + * Manages voice interaction state transitions. + * Enforces valid transitions and notifies observers. + */ +class VoiceStateMachine { + + private val _state = MutableStateFlow(VoiceState.IDLE) + val state: StateFlow = _state.asStateFlow() + + fun transition(newState: VoiceState): Boolean { + val current = _state.value + if (!isValidTransition(current, newState)) return false + _state.value = newState + return true + } + + private fun isValidTransition(from: VoiceState, to: VoiceState): Boolean { + return when (from) { + VoiceState.IDLE -> to == VoiceState.LISTENING + VoiceState.LISTENING -> to in setOf(VoiceState.PROCESSING, VoiceState.IDLE) + VoiceState.PROCESSING -> to in setOf(VoiceState.RECEIVING, VoiceState.IDLE) + VoiceState.RECEIVING -> to in setOf(VoiceState.SPEAKING, VoiceState.IDLE) + VoiceState.SPEAKING -> to in setOf(VoiceState.IDLE, VoiceState.LISTENING) // LISTENING = interruption + } + } + + fun reset() { + _state.value = VoiceState.IDLE + } +} diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml new file mode 100644 index 0000000..7d795d8 --- /dev/null +++ b/app/src/main/res/values/strings.xml @@ -0,0 +1,4 @@ + + + ClippyAuto + diff --git a/app/src/main/res/values/themes.xml b/app/src/main/res/values/themes.xml new file mode 100644 index 0000000..95a0809 --- /dev/null +++ b/app/src/main/res/values/themes.xml @@ -0,0 +1,8 @@ + + + + diff --git a/app/src/main/res/xml/automotive_app_desc.xml b/app/src/main/res/xml/automotive_app_desc.xml new file mode 100644 index 0000000..0fb852c --- /dev/null +++ b/app/src/main/res/xml/automotive_app_desc.xml @@ -0,0 +1,4 @@ + + + + diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..a60f7c9 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,5 @@ +plugins { + id("com.android.application") version "8.7.3" apply false + id("org.jetbrains.kotlin.android") version "2.1.10" apply false + id("com.google.gms.google-services") version "4.4.2" apply false +} diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md new file mode 100644 index 0000000..5121009 --- /dev/null +++ b/docs/ARCHITECTURE.md @@ -0,0 +1,298 @@ +# ClippyAuto πŸ“ŽπŸš— β€” Final Spec (3 Rounds Complete) + +## Vision +A lightweight Android app purpose-built for voice-first interaction with Clippy via Matrix, optimized for Android Auto commute use. + +## Core User Story +> As Jeremy, I get in my car, Android Auto launches, I tap "Clippy" (or it auto-starts), and I'm in a live voice conversation with my AI Chief of Staff. I talk, Clippy responds in my car speakers within seconds. No typing, no "Hey Google," no Element. + +--- + +## Locked Decisions + +| Decision | Resolution | Source | +|---|---|---| +| Language | Kotlin | Spirit R1 | +| Matrix SDK | Raw REST API (Ktor) β€” no E2EE needed for private homeserver | Spirit R3 | +| STT | Android SpeechRecognizer (Vosk offline fallback in v2) | Spirit R1 | +| TTS | Android native TextToSpeech | Spirit R1 | +| Audio focus | AUDIOFOCUS_GAIN_TRANSIENT_MAY_DUCK / USAGE_ASSISTANT | Spirit R1 | +| Auth | Access token in Android Keystore | Spirit R1 | +| Push delivery | FCM via n8n + Matrix sync dual-path, first wins | Clippy R2 | +| Music integration | None β€” Android handles ducking | Margaret R2 | +| Private mode | Voice command "go quiet" / "speak" | Margaret R2 | +| Interruption | 200ms TTS fade, activate STT, cancel queued chunks | Clippy R2 | +| Delegation | Always confirm verbally before sending | Clippy R2 | +| Audio cue (heard you) | Subtle chime at 2s silence mark, not voice | Clippy R2 | +| Thinking indicator | Low ambient tone after 3s processing | Clippy R2 | +| Session resume | < 15 min = resume, > 15 min = new session | Margaret R2, Clippy R2 | +| Sentence chunking | v1 scope β€” each sentence as separate Matrix message | Clippy R2 | +| Dedicated room | #clippy-voice with custom event fields | Margaret R2 | +| E2EE | Skip for v1 (private homeserver, invite-only room) | Clippy R3 | + +--- + +## Functional Requirements + +### FR-1: Voice Input +- On-device STT via Android SpeechRecognizer +- Push-to-talk button on Android Auto screen (large, driver-safe) +- 2-second silence threshold with subtle confirmation chime +- Cancel-send if Jeremy resumes talking within 1s of chime + +### FR-2: Matrix Integration +- Raw REST API client (Ktor) β€” login, sync, send +- Authenticates as Jeremy (dedicated device token) +- Sends transcribed text to #clippy-voice room +- Custom event fields: `farm.hobbs.clippy.source`, `farm.hobbs.clippy.session_id`, `farm.hobbs.clippy.signal` + +### FR-3: Voice Output +- Android native TextToSpeech +- Plays through car audio via USAGE_ASSISTANT audio attributes +- Queues chunked responses, plays sequentially +- 200ms fade on interruption (not hard cut) + +### FR-4: Android Auto Integration +- CarAppService with custom Screen template +- Minimal UI: status indicator (listening/thinking/speaking), talk button +- Auto-launch option when Android Auto connects (FR-6) +- Complies with driver distraction guidelines + +### FR-5: Session Awareness +- session_start signal on app open β†’ triggers arrival briefing +- session_end signal on close/timeout β†’ triggers close summary +- session_resume for reconnects < 15 min + +### FR-6: Auto-Launch +- Start ClippyAuto automatically when Android Auto connects + +### FR-7: Session Context Injection +- App sends `[ClippyAuto session started β€” commute mode, voice-first]` to Matrix room +- Clippy shifts to debrief mode automatically + +### FR-8: Debrief Summary +- On session end, Clippy posts written summary (formatted, markdown OK) +- Phone UI surfaces as notification: "Debrief summary ready" + +### FR-9: Voice Commands +| Command | Action | +|---|---| +| "Clippy, go quiet" | Mute TTS, switch to phone text display | +| "Clippy, I'm back" / "speak" | Resume voice mode | +| "Clippy, wrap it up" | Trigger session close + summary | +| "Clippy, remind me [thing]" | Create reminder in memory | +| "Clippy, tell [agent] [message]" | Delegation with verbal confirmation | + +### FR-10: Silence Detection Tuning +- 2s pause = processing trigger +- Subtle confirmation chime at 2s mark +- If speech resumes within 1s of chime, cancel send and keep listening + +### FR-11: Response Length Guardrails +- App-side: max 250 chars per TTS chunk +- Split responses > 500 chars into segments with brief pause +- Clippy enforces 2-sentence max per message in voice mode + +--- + +## Non-Functional Requirements + +| ID | Requirement | Target | +|---|---|---| +| NFR-1 | Time to first audio | < 5 seconds (with FCM + chunking) | +| NFR-2 | Offline grace | Queue messages, notify verbally, resume on reconnect | +| NFR-3 | Battery | Minimal drain; only active in foreground | +| NFR-4 | Privacy | All voice processing on-device; only text hits network | +| NFR-5 | Eyes-free | Zero required visual interactions after launch | + +--- + +## Architecture + +``` +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ Android Auto Screen β”‚ +β”‚ [ πŸŽ™οΈ Talk to Clippy ] [Status] β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ +β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” +β”‚ ClippyAuto App β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ STT Engineβ”‚ β”‚ TTS Engine β”‚ β”‚ +β”‚ β”‚ (on-device)β”‚ β”‚ (on-device) β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β”€β”€β”€β–²β”€β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β”‚ β”‚ text β”‚ text β”‚ +β”‚ β”Œβ”€β”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β”€β”€β” β”‚ +β”‚ β”‚ Matrix REST Client (Ktor) β”‚ β”‚ +β”‚ β”‚ Room: #clippy-voice β”‚ β”‚ +β”‚ β”‚ + FCM receiver (fast path) β”‚ β”‚ +β”‚ β””β”€β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–²β”€β”€β”€β”€β”€β”€β”€β”˜ β”‚ +β””β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”Œβ”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β” + β”‚ mess.hobbs.farm:8008 β”‚ + β”‚ Matrix Synapse β”‚ + β”‚ β”‚ β”‚ + β”‚ Push gateway β†’ n8n β”‚ + β”‚ β”‚ β”‚ + β”‚ β–Ό β”‚ + β”‚ FCM data message β”‚ + β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β–²β”€β”€β”€β”€β”˜ + β”‚ β”‚ + β”Œβ”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”΄β”€β”€β”€β”€β” + β”‚ Clippy (OpenClaw Agent) β”‚ + β”‚ Claude β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +### Push Path (Fast) +``` +Clippy sends message β†’ Synapse push gateway β†’ n8n webhook + β†’ n8n filters (sender == @clippy) β†’ FCM data message β†’ app β†’ TTS +Latency: ~400-700ms from Clippy message to TTS start +``` + +### Sync Path (Fallback) +``` +Clippy sends message β†’ Matrix sync long-poll β†’ app β†’ TTS +Latency: ~1-2s (depends on poll timing) +``` + +Both paths active; first to deliver wins; dedup via hash. + +--- + +## Latency Budget + +| Step | Target | Notes | +|---|---|---| +| STT | 800ms | On-device | +| Matrix send | 400ms | Cellular | +| Claude to first sentence | 1.5-2.5s | Sentence chunking | +| FCM delivery | 300ms | High priority data message | +| TTS start | 300ms | Pre-initialized | +| **Total to first audio** | **3.3-4.3s** | 🎯 Under 5 seconds | + +--- + +## Voice State Machine + +``` + β”Œβ”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β” + β”‚ IDLE │◄──────────────────────┐ + β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β”‚ + β”‚ tap button β”‚ TTS complete + β”Œβ”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β” + β”‚LISTENING β”‚ β”‚ SPEAKING β”‚ + β””β”€β”€β”€β”€β”¬β”€β”€β”€β”€β”€β”˜ β””β”€β”€β”€β”€β–²β”€β”€β”€β”€β”€β”˜ + β”‚ 2s silence + chime β”‚ response received + β”Œβ”€β”€β”€β”€β–Όβ”€β”€β”€β”€β”€β”€β” β”Œβ”€β”€β”€β”€β”΄β”€β”€β”€β”€β”€β”€β” + β”‚PROCESSING │────────────────►│ RECEIVING β”‚ + β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ sent to Matrix β””β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ +``` + +--- + +## Connectivity Handling + +| Scenario | Behavior | +|---|---| +| Tunnel / dead zone | Queue outgoing, TTS: "Signal dropped, I'll catch up." | +| Clippy slow (>10s) | TTS: "Still thinking on that one..." | +| Bluetooth disconnect | Pause session, resume on reconnect | +| Resume < 15 min | session_resume, "Welcome back" | +| Resume > 15 min | session_start, fresh arrival briefing | + +--- + +## Clippy Voice Mode Prompt (appended to SOUL.md during voice sessions) + +``` +VOICE MODE RULES: +1. Max 2 sentences per message. Hard limit. +2. Send each sentence as a separate Matrix message (enables chunked TTS). +3. No markdown. No bullet points. No formatting. Plain speech only. +4. No emoji in voice responses. +5. Numbers: say "three" not "3". Say "two thirty" not "2:30 PM". +6. Names: full name first mention, first name after. +7. Don't say "let me think about that" β€” just think and respond. +8. Lists conversationally: "You've got three things: X, Y, and Z." + +ARRIVAL BRIEFING (on session_start): + Message 1: "Hey Jeremy. [Most urgent/time-sensitive thing]." + Message 2: "[One open loop from last session or overnight change]." + Then: silence. + +SESSION CLOSE (on "wrap it up" or session_end): + Message 1: "Alright, here's what I've got. [Action items conversationally]." + Message 2: "[Anything unresolved carrying to next session]." + Then: post written markdown summary to room. + +SESSION RESUME (on session_resume): + Message 1: "Welcome back. [Where we left off, one sentence]." + Then: silence. +``` + +--- + +## Security + +| Concern | Mitigation | +|---|---| +| Token storage | Android Keystore (hardware-backed) | +| Network | TLS to homeserver | +| FCM payload | Data message only, no sensitive text in notification | +| Voice data | Never stored; STT real-time, audio buffer discarded | +| Phone screen | Locked during Android Auto; transcript after session | + +--- + +## v1 MVP Scope + +**Ships:** +- Push-to-talk Android Auto interface +- On-device STT β†’ Matrix REST β†’ Clippy β†’ FCM β†’ TTS +- Sentence-level chunked responses (sub-5s first audio) +- Arrival briefing, session close summary +- Voice commands (quiet/speak/wrap up/remind/delegate) +- Private/passenger mode +- Connectivity handling +- Dedicated #clippy-voice Matrix room +- Minimal phone UI (setup + session active) + +**v2 Backlog:** +- Wake word ("Hey Clippy") via Porcupine +- Google Calendar integration +- Direct SSE streaming endpoint +- Offline STT (Vosk fallback) +- Full phone transcript review UI +- Agent overnight digest system + +--- + +## Build Estimate + +| Component | Days | +|---|---| +| Project setup + Android Auto scaffold | 2 | +| Matrix REST client + auth | 1.5 | +| Voice pipeline (STT + TTS + state machine) | 3 | +| FCM push path (app side) | 1 | +| n8n workflow (server side) | 0.5 | +| Clippy voice mode prompt tuning | 1 | +| Android Auto UI | 2 | +| Phone UI (minimal) | 1 | +| Testing + polish | 3 | +| **Total** | **~15 days** | + +--- + +## Open Questions for Jeremy + +1. **Build approach** β€” Who builds this? Spirit scaffolds? Jeremy builds with our spec? Fork existing project? +2. **Wake word** β€” "Hey Clippy" for v2, or push-to-talk fine indefinitely? +3. **Which Google Calendar(s)** for v2? +4. **Agent overnight reports** β€” Want Margaret/Spirit posting daily summaries for Clippy to pull from? +5. **Homeserver external access** β€” Is mess.hobbs.farm already reachable via Traefik externally? +6. **Firebase project** β€” Existing one or create new? +7. **Testing** β€” Wired Android Auto first, or straight to wireless? diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..7d0c0db --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,17 @@ +pluginManagement { + repositories { + google() + mavenCentral() + gradlePluginPortal() + } +} + +dependencyResolution { + repositories { + google() + mavenCentral() + } +} + +rootProject.name = "ClippyAuto" +include(":app")